Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix camera limitation due to jpg encoding #171

Merged
merged 3 commits into from
Oct 7, 2024
Merged

Fix camera limitation due to jpg encoding #171

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d1da2d0
Switch compressed image to raw image
alexandrefch Sep 24, 2024
be0848c
Remove 'compressed' from camera topic name in camera prefab
alexandrefch Oct 7, 2024
20a27b1
Merge branch 'beta/v2.x.x' into fix/camera_image_jpg_limitation
alexandrefch Oct 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions Assets/UnitySensors/Runtime/Scripts/Data/Texture/ITextureInterface.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,5 +5,9 @@ namespace UnitySensors.Data.Texture
public interface ITextureInterface
{
public Texture2D texture { get; }
public byte[] data { get; }
public int width { get; }
public int height { get; }
public string encoding { get; }
}
}
54 changes: 42 additions & 12 deletions Assets/UnitySensors/Runtime/Scripts/Sensors/Camera/CameraSensor.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,17 @@ public abstract class CameraSensor : UnitySensor, ITextureInterface
private UnityEngine.Camera _m_camera;
private RenderTexture _rt = null;
private Texture2D _texture;
private byte[] _data;

public Vector2Int resolution { get => _resolution; }
protected float maxRange { get => _maxRange; }
public UnityEngine.Camera m_camera { get => _m_camera; }
public Texture2D texture { get => _texture; }
public byte[] data { get => _data; }
public int width { get => _resolution.x; }
public int height { get => _resolution.y; }
public string encoding { get => "rgb8"; }


protected override void Init()
{
Expand All @@ -33,32 +39,50 @@ protected override void Init()
_m_camera.nearClipPlane = _minRange;
_m_camera.farClipPlane = _maxRange;

_rt = new RenderTexture(_resolution.x, _resolution.y, 32, RenderTextureFormat.ARGBFloat);
_rt = new RenderTexture(_resolution.x, _resolution.y, 16, RenderTextureFormat.ARGB32);
_m_camera.targetTexture = _rt;

_texture = new Texture2D(_resolution.x, _resolution.y, TextureFormat.RGBAFloat, false);
_texture = new Texture2D(width, height, TextureFormat.RGB24, false);

_data = new byte[width * height * 3];
}

protected override void UpdateSensor()
{
if (!LoadTexture()) return;

if (onSensorUpdated != null)
onSensorUpdated.Invoke();
}

protected bool LoadTexture()
{
bool result = false;
AsyncGPUReadback.Request(_rt, 0, request => {
if (request.hasError)
{
}
else
{
var data = request.GetData<Color>();
_texture.LoadRawTextureData(data);

// Blit to a temporary texture and request readback on it.
AsyncGPUReadback.Request(_rt, 0, TextureFormat.ARGB32, request => {
if (request.hasError) {
Debug.LogWarning("GPU readback error was detected.");
} else {
var dataBuffer = request.GetData<byte>();

// Flip image
int i=0;
int j=width*(height-1)*4;
for(int y=0; y<height; y++){
for(int x=0; x<width; x++) {
_data[i+0] = dataBuffer[j+1];
_data[i+1] = dataBuffer[j+2];
_data[i+2] = dataBuffer[j+3];
i+=3;
j+=4;
}
j -= width << 3; // width * 2 * 3;
}

_texture.LoadRawTextureData(_data);
_texture.Apply();

result = true;
}
});
Expand All @@ -68,7 +92,13 @@ protected bool LoadTexture()

protected override void OnSensorDestroy()
{
_rt.Release();
if (_rt != null)
{
// Dispose the frame texture.
m_camera.targetTexture = null;
Destroy(_rt);
_rt = null;
}
}
}
}
2 changes: 1 addition & 1 deletion Assets/UnitySensorsROS/Runtime/Scripts/Publishers/Image/ImageMsgPublisher.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

namespace UnitySensors.ROS.Publisher.Image
{
public class ImageMsgPublisher<T> : RosMsgPublisher<T, ImageMsgSerializer<T>, CompressedImageMsg> where T : UnitySensor, ITextureInterface
public class ImageMsgPublisher<T> : RosMsgPublisher<T, ImageMsgSerializer<T>, ImageMsg> where T : UnitySensor, ITextureInterface
{
}
}
14 changes: 8 additions & 6 deletions Assets/UnitySensorsROS/Runtime/Scripts/Serializers/Image/ImageMsgSerializer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,25 +7,27 @@
namespace UnitySensors.ROS.Serializer.Image
{
[System.Serializable]
public class ImageMsgSerializer<T> : RosMsgSerializer<T, CompressedImageMsg> where T : UnitySensor, ITextureInterface
public class ImageMsgSerializer<T> : RosMsgSerializer<T, ImageMsg> where T : UnitySensor, ITextureInterface
{
[SerializeField]
private HeaderSerializer _header;
[SerializeField, Range(1, 100)]
private int quality = 75;

public override void Init(T sensor)
{
base.Init(sensor);
_header.Init(sensor);

_msg.format = "jpeg";
_msg.height = (uint)sensor.height;
_msg.width = (uint)sensor.width;
_msg.encoding = sensor.encoding;
_msg.is_bigendian = 0;
_msg.step = 1 * 3 * _msg.width;
}

public override CompressedImageMsg Serialize()
public override ImageMsg Serialize()
{
_msg.header = _header.Serialize();
_msg.data = sensor.texture.EncodeToJPG(quality);
_msg.data = sensor.data;
return _msg;
}
}
Expand Down
Loading