Skip to content

Commit 830ff76

Browse files
github-actions[bot]yuokamotobresilla
authored
Ue5.3 action update (#354) (#357)
* feat: support actual depth image with 32FC1 datatype (#349) - Add separate processing for RGB and Depth camera types with specific capture source and image properties. - Introduce new data structures to handle RGB and Depth surface contexts. - Adjust pixel format and data handling based on camera type, including custom encoding and image data initialization. - Improve conditions for processing captured data by ensuring proper checks are in place for RGB and Depth data retrieval. - Add a `Depth` array to `FRenderRequest` struct to store depth information. - Remove obsolete comments and redundant code sections to improve readability and maintainability. * update action --------- Co-authored-by: yuokamoto <[email protected]> Co-authored-by: Trim Bresilla <[email protected]>
1 parent a08a68c commit 830ff76

File tree

3 files changed

+200
-159
lines changed

3 files changed

+200
-159
lines changed

‎.github/workflows/main.yml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,9 @@ on:
66
- devel
77
- UE5.2
88
- UE5.3
9+
- UE5.4
10+
- UE5.4.4
11+
- UE5.5
912
types: ["closed"]
1013

1114
workflow_dispatch:
@@ -25,12 +28,21 @@ jobs:
2528
- devel
2629
- UE5.2
2730
- UE5.3
31+
- UE5.4
32+
- UE5.4.4
33+
- UE5.5
2834
steps:
35+
- name: Check token permissions
36+
run: |
37+
curl -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
38+
-H "Accept: application/vnd.github.v3+json" \
39+
https://api.github.com/repos/${{ github.repository }}
2940
- name: Checkout
3041
uses: actions/checkout@v4
3142
if: matrix.version != github.base_ref
3243
with:
3344
ref: ${{ matrix.version }}
45+
token: ${{ secrets.GITHUB_TOKEN }}
3446
- name: Debug
3547
env:
3648
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

‎Source/RapyutaSimulationPlugins/Private/Sensors/RRROS2CameraComponent.cpp‎

Lines changed: 184 additions & 156 deletions
Original file line numberDiff line numberDiff line change
@@ -1,156 +1,184 @@
1-
// Copyright 2020-2023 Rapyuta Robotics Co., Ltd.
2-
3-
#include "Sensors/RRROS2CameraComponent.h"
4-
5-
#include "BufferVisualizationData.h"
6-
7-
URRROS2CameraComponent::URRROS2CameraComponent()
8-
{
9-
// component initialization
10-
SceneCaptureComponent = CreateDefaultSubobject<USceneCaptureComponent2D>(TEXT("SceneCaptureComponent"));
11-
SceneCaptureComponent->SetupAttachment(this);
12-
13-
CameraComponent = CreateDefaultSubobject<UCameraComponent>(TEXT("CameraComponent"));
14-
CameraComponent->SetupAttachment(this);
15-
16-
TopicName = TEXT("raw_image");
17-
MsgClass = UROS2ImgMsg::StaticClass();
18-
}
19-
20-
void URRROS2CameraComponent::PreInitializePublisher(UROS2NodeComponent* InROS2Node, const FString& InTopicName)
21-
{
22-
SceneCaptureComponent->FOVAngle = CameraComponent->FieldOfView;
23-
SceneCaptureComponent->OrthoWidth = CameraComponent->OrthoWidth;
24-
25-
if (CameraType == EROS2CameraType::DEPTH)
26-
{
27-
FWeightedBlendable blendable(1.0f, GetBufferVisualizationData().GetMaterial(TEXT("SceneDepth")));
28-
CameraComponent->PostProcessSettings.WeightedBlendables.Array.Add(blendable);
29-
SceneCaptureComponent->PostProcessSettings = CameraComponent->PostProcessSettings;
30-
SceneCaptureComponent->CaptureSource = ESceneCaptureSource::SCS_FinalColorHDR;
31-
}
32-
33-
RenderTarget = NewObject<UTextureRenderTarget2D>(this, UTextureRenderTarget2D::StaticClass());
34-
RenderTarget->InitCustomFormat(Width, Height, EPixelFormat::PF_B8G8R8A8, true);
35-
SceneCaptureComponent->TextureTarget = RenderTarget;
36-
37-
// Initialize image data
38-
Data.Header.FrameId = FrameId;
39-
Data.Width = Width;
40-
Data.Height = Height;
41-
Data.Encoding = Encoding;
42-
Data.Step = Width * 3; // todo should be variable based on encoding
43-
Data.Data.AddUninitialized(Width * Height * 3);
44-
45-
QueueSize = QueueSize < 1 ? 1 : QueueSize; // QueueSize should be more than 1
46-
47-
Super::PreInitializePublisher(InROS2Node, InTopicName);
48-
}
49-
50-
void URRROS2CameraComponent::SensorUpdate()
51-
{
52-
SceneCaptureComponent->CaptureScene();
53-
CaptureNonBlocking();
54-
}
55-
56-
// reference https://github.com/TimmHess/UnrealImageCapture
57-
void URRROS2CameraComponent::CaptureNonBlocking()
58-
{
59-
SceneCaptureComponent->TextureTarget->TargetGamma = GEngine->GetDisplayGamma();
60-
// Get RenderContext
61-
FTextureRenderTargetResource* renderTargetResource = SceneCaptureComponent->TextureTarget->GameThread_GetRenderTargetResource();
62-
63-
struct FReadSurfaceContext
64-
{
65-
FRenderTarget* SrcRenderTarget;
66-
TArray<FColor>* OutData;
67-
FIntRect Rect;
68-
FReadSurfaceDataFlags Flags;
69-
};
70-
71-
// Init new RenderRequest
72-
FRenderRequest* renderRequest = new FRenderRequest();
73-
74-
// Setup GPU command
75-
FReadSurfaceContext readSurfaceContext = {
76-
renderTargetResource,
77-
&(renderRequest->Image),
78-
FIntRect(0, 0, renderTargetResource->GetSizeXY().X, renderTargetResource->GetSizeXY().Y),
79-
FReadSurfaceDataFlags(RCM_UNorm, CubeFace_MAX)};
80-
81-
// Above 4.22 use this
82-
ENQUEUE_RENDER_COMMAND(SceneDrawCompletion)
83-
(
84-
[readSurfaceContext, this](FRHICommandListImmediate& RHICmdList)
85-
{
86-
RHICmdList.ReadSurfaceData(readSurfaceContext.SrcRenderTarget->GetRenderTargetTexture(),
87-
readSurfaceContext.Rect,
88-
*readSurfaceContext.OutData,
89-
readSurfaceContext.Flags);
90-
});
91-
92-
// Notify new task in RenderQueue
93-
RenderRequestQueue.Enqueue(renderRequest);
94-
if (QueueCount > QueueSize)
95-
{
96-
RenderRequestQueue.Pop();
97-
}
98-
else
99-
{
100-
QueueCount++;
101-
}
102-
103-
// Set RenderCommandFence
104-
renderRequest->RenderFence.BeginFence();
105-
}
106-
107-
FROSImg URRROS2CameraComponent::GetROS2Data()
108-
{
109-
if (!RenderRequestQueue.IsEmpty())
110-
{
111-
// Timestamp
112-
Data.Header.Stamp = URRConversionUtils::FloatToROSStamp(UGameplayStatics::GetTimeSeconds(GetWorld()));
113-
114-
// Peek the next RenderRequest from queue
115-
FRenderRequest* nextRenderRequest = nullptr;
116-
RenderRequestQueue.Peek(nextRenderRequest);
117-
if (nextRenderRequest)
118-
{ // nullptr check
119-
if (nextRenderRequest->RenderFence.IsFenceComplete())
120-
{ // Check if rendering is done, indicated by RenderFence
121-
for (int I = 0; I < nextRenderRequest->Image.Num(); I++)
122-
{
123-
Data.Data[I * 3 + 0] = nextRenderRequest->Image[I].R;
124-
Data.Data[I * 3 + 1] = nextRenderRequest->Image[I].G;
125-
Data.Data[I * 3 + 2] = nextRenderRequest->Image[I].B;
126-
}
127-
128-
// Delete the first element from RenderQueue
129-
RenderRequestQueue.Pop();
130-
QueueCount--;
131-
delete nextRenderRequest;
132-
}
133-
}
134-
}
135-
136-
// SceneCaptureComponent->CaptureScene();
137-
// FTextureRenderTarget2DResource* RenderTargetResource;
138-
// RenderTargetResource = (FTextureRenderTarget2DResource*)RenderTarget->GameThread_GetRenderTargetResource();
139-
// if (RenderTargetResource) {
140-
// TArray<FColor> buffer;
141-
// RenderTargetResource->ReadPixels(buffer);
142-
// for (int I = 0; I < buffer.Num(); I++)
143-
// {
144-
// Data.data[I * 3 + 0] = buffer[I].R;
145-
// Data.data[I * 3 + 1] = buffer[I].G;
146-
// Data.data[I * 3 + 2] = buffer[I].B;
147-
// }
148-
// }
149-
150-
return Data;
151-
}
152-
153-
void URRROS2CameraComponent::SetROS2Msg(UROS2GenericMsg* InMessage)
154-
{
155-
CastChecked<UROS2ImgMsg>(InMessage)->SetMsg(GetROS2Data());
156-
}
1+
// Copyright 2020-2023 Rapyuta Robotics Co., Ltd.
2+
3+
#include "Sensors/RRROS2CameraComponent.h"
4+
5+
#include "BufferVisualizationData.h"
6+
7+
URRROS2CameraComponent::URRROS2CameraComponent()
8+
{
9+
// component initialization
10+
SceneCaptureComponent = CreateDefaultSubobject<USceneCaptureComponent2D>(TEXT("SceneCaptureComponent"));
11+
SceneCaptureComponent->SetupAttachment(this);
12+
13+
CameraComponent = CreateDefaultSubobject<UCameraComponent>(TEXT("CameraComponent"));
14+
CameraComponent->SetupAttachment(this);
15+
16+
TopicName = TEXT("raw_image");
17+
MsgClass = UROS2ImgMsg::StaticClass();
18+
}
19+
20+
void URRROS2CameraComponent::PreInitializePublisher(UROS2NodeComponent* InROS2Node, const FString& InTopicName)
21+
{
22+
SceneCaptureComponent->FOVAngle = CameraComponent->FieldOfView;
23+
SceneCaptureComponent->OrthoWidth = CameraComponent->OrthoWidth;
24+
25+
// Set capture source and image properties based on camera type
26+
if (CameraType == EROS2CameraType::RGB)
27+
{
28+
SceneCaptureComponent->CaptureSource = ESceneCaptureSource::SCS_FinalColorHDR;
29+
RenderTarget = NewObject<UTextureRenderTarget2D>(this, UTextureRenderTarget2D::StaticClass());
30+
RenderTarget->InitCustomFormat(Width, Height, EPixelFormat::PF_B8G8R8A8, true);
31+
32+
Data.Encoding = TEXT("bgr8");
33+
Data.Step = Width * 3;
34+
}
35+
else if (CameraType == EROS2CameraType::DEPTH)
36+
{
37+
CameraComponent->PostProcessSettings.WeightedBlendables.Array.Add(
38+
FWeightedBlendable(1.0f, GetBufferVisualizationData().GetMaterial(TEXT("SceneDepth"))));
39+
SceneCaptureComponent->PostProcessSettings = CameraComponent->PostProcessSettings;
40+
SceneCaptureComponent->CaptureSource = ESceneCaptureSource::SCS_SceneDepth;
41+
42+
RenderTarget = NewObject<UTextureRenderTarget2D>(this, UTextureRenderTarget2D::StaticClass());
43+
RenderTarget->InitCustomFormat(Width, Height, EPixelFormat::PF_FloatRGBA, false);
44+
Data.Encoding = TEXT("32FC1");
45+
Data.Step = Width * 4;
46+
}
47+
48+
// Common setup
49+
SceneCaptureComponent->TextureTarget = RenderTarget;
50+
Data.Header.FrameId = FrameId;
51+
Data.Width = Width;
52+
Data.Height = Height;
53+
Data.Data.AddUninitialized(Width * Height * (CameraType == EROS2CameraType::RGB ? 3 : 4));
54+
QueueSize = QueueSize < 1 ? 1 : QueueSize; // QueueSize should be more than 1
55+
Super::PreInitializePublisher(InROS2Node, InTopicName);
56+
}
57+
58+
void URRROS2CameraComponent::SensorUpdate()
59+
{
60+
if (Render)
61+
{
62+
SceneCaptureComponent->CaptureScene();
63+
CaptureNonBlocking();
64+
}
65+
}
66+
67+
// reference https://github.com/TimmHess/UnrealImageCapture
68+
void URRROS2CameraComponent::CaptureNonBlocking()
69+
{
70+
SceneCaptureComponent->TextureTarget->TargetGamma = GEngine->GetDisplayGamma();
71+
// Get RenderContext
72+
FTextureRenderTargetResource* renderTargetResource = SceneCaptureComponent->TextureTarget->GameThread_GetRenderTargetResource();
73+
74+
struct FReadSurfaceContextRGB
75+
{
76+
FRenderTarget* SrcRenderTarget;
77+
TArray<FColor>* OutData;
78+
FIntRect Rect;
79+
FReadSurfaceDataFlags Flags;
80+
};
81+
82+
struct FReadSurfaceContextDepth
83+
{
84+
FRenderTarget* SrcRenderTarget;
85+
TArray<FFloat16Color>* Depth;
86+
FIntRect Rect;
87+
FReadSurfaceDataFlags Flags;
88+
};
89+
90+
// Init new RenderRequest
91+
FRenderRequest* renderRequest = new FRenderRequest();
92+
93+
// Setup GPU command
94+
FIntRect rect(0, 0, renderTargetResource->GetSizeXY().X, renderTargetResource->GetSizeXY().Y);
95+
FReadSurfaceDataFlags flags(RCM_UNorm, CubeFace_MAX);
96+
97+
if (CameraType == EROS2CameraType::RGB)
98+
{
99+
FReadSurfaceContextRGB readSurfaceContext = {renderTargetResource, &(renderRequest->Image), rect, flags};
100+
101+
ENQUEUE_RENDER_COMMAND(SceneDrawCompletion)
102+
(
103+
[readSurfaceContext, this](FRHICommandListImmediate& RHICmdList)
104+
{
105+
RHICmdList.ReadSurfaceData(readSurfaceContext.SrcRenderTarget->GetRenderTargetTexture(),
106+
readSurfaceContext.Rect,
107+
*readSurfaceContext.OutData,
108+
readSurfaceContext.Flags);
109+
});
110+
}
111+
else if (CameraType == EROS2CameraType::DEPTH)
112+
{
113+
FReadSurfaceContextDepth readSurfaceContext = {renderTargetResource, &(renderRequest->Depth), rect, flags};
114+
115+
ENQUEUE_RENDER_COMMAND(SceneDrawCompletion)
116+
(
117+
[readSurfaceContext, this](FRHICommandListImmediate& RHICmdList)
118+
{
119+
RHICmdList.ReadSurfaceFloatData(readSurfaceContext.SrcRenderTarget->GetRenderTargetTexture(),
120+
readSurfaceContext.Rect,
121+
*readSurfaceContext.Depth,
122+
readSurfaceContext.Flags);
123+
});
124+
}
125+
126+
// Notify new task in RenderQueue
127+
RenderRequestQueue.Enqueue(renderRequest);
128+
if (QueueCount > QueueSize)
129+
{
130+
RenderRequestQueue.Pop();
131+
}
132+
else
133+
{
134+
QueueCount++;
135+
}
136+
137+
// Set RenderCommandFence
138+
renderRequest->RenderFence.BeginFence();
139+
}
140+
141+
FROSImg URRROS2CameraComponent::GetROS2Data()
142+
{
143+
if (!RenderRequestQueue.IsEmpty())
144+
{
145+
// Timestamp
146+
Data.Header.Stamp = URRConversionUtils::FloatToROSStamp(UGameplayStatics::GetTimeSeconds(GetWorld()));
147+
// Peek the next RenderRequest from queue
148+
FRenderRequest* nextRenderRequest = nullptr;
149+
RenderRequestQueue.Peek(nextRenderRequest);
150+
if (nextRenderRequest && nextRenderRequest->RenderFence.IsFenceComplete())
151+
{
152+
if (CameraType == EROS2CameraType::RGB)
153+
{
154+
// Process RGB data
155+
for (int i = 0; i < nextRenderRequest->Image.Num(); i++)
156+
{
157+
Data.Data[i * 3 + 0] = nextRenderRequest->Image[i].B;
158+
Data.Data[i * 3 + 1] = nextRenderRequest->Image[i].G;
159+
Data.Data[i * 3 + 2] = nextRenderRequest->Image[i].R;
160+
}
161+
}
162+
else if (CameraType == EROS2CameraType::DEPTH)
163+
{
164+
// Process Depth data
165+
for (int i = 0; i < nextRenderRequest->Depth.Num(); i++)
166+
{
167+
float value = nextRenderRequest->Depth[i].R.GetFloat() / 100;
168+
std::memcpy(&Data.Data[i * 4], &value, sizeof(value));
169+
}
170+
}
171+
172+
// Delete the first element from RenderQueue
173+
RenderRequestQueue.Pop();
174+
QueueCount--;
175+
delete nextRenderRequest;
176+
}
177+
}
178+
return Data;
179+
}
180+
181+
void URRROS2CameraComponent::SetROS2Msg(UROS2GenericMsg* InMessage)
182+
{
183+
CastChecked<UROS2ImgMsg>(InMessage)->SetMsg(GetROS2Data());
184+
}

‎Source/RapyutaSimulationPlugins/Public/Sensors/RRROS2CameraComponent.h‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ struct FRenderRequest
3030
{
3131
GENERATED_BODY()
3232
TArray<FColor> Image;
33+
TArray<FFloat16Color> Depth;
3334
FRenderCommandFence RenderFence;
3435
};
3536

@@ -114,6 +115,9 @@ class RAPYUTASIMULATIONPLUGINS_API URRROS2CameraComponent : public URRROS2BaseSe
114115
UPROPERTY(EditAnywhere, BlueprintReadWrite)
115116
EROS2CameraType CameraType = EROS2CameraType::RGB;
116117

118+
UPROPERTY(EditAnywhere, BlueprintReadWrite)
119+
bool Render = true;
120+
117121
// ROS
118122
/**
119123
* @brief Update ROS 2 Msg structure from #RenderRequestQueue
@@ -129,7 +133,4 @@ class RAPYUTASIMULATIONPLUGINS_API URRROS2CameraComponent : public URRROS2BaseSe
129133
* @param InMessage
130134
*/
131135
virtual void SetROS2Msg(UROS2GenericMsg* InMessage) override;
132-
133-
UPROPERTY(EditAnywhere, BlueprintReadWrite)
134-
FString Encoding = TEXT("rgb8");
135136
};

0 commit comments

Comments
 (0)