diff --git a/Content/Materials/PP_Segmentation.uasset b/Content/Materials/PP_Segmentation.uasset new file mode 100644 index 00000000..5db2eb55 Binary files /dev/null and b/Content/Materials/PP_Segmentation.uasset differ diff --git a/Source/RapyutaSimulationPlugins/Private/Sensors/RRROS2CameraComponent.cpp b/Source/RapyutaSimulationPlugins/Private/Sensors/RRROS2CameraComponent.cpp index 7358cb74..f6da4b49 100644 --- a/Source/RapyutaSimulationPlugins/Private/Sensors/RRROS2CameraComponent.cpp +++ b/Source/RapyutaSimulationPlugins/Private/Sensors/RRROS2CameraComponent.cpp @@ -1,7 +1,6 @@ // Copyright 2020-2023 Rapyuta Robotics Co., Ltd. #include "Sensors/RRROS2CameraComponent.h" - #include "BufferVisualizationData.h" URRROS2CameraComponent::URRROS2CameraComponent() @@ -21,28 +20,56 @@ void URRROS2CameraComponent::PreInitializePublisher(UROS2NodeComponent* InROS2No { SceneCaptureComponent->FOVAngle = CameraComponent->FieldOfView; SceneCaptureComponent->OrthoWidth = CameraComponent->OrthoWidth; + SceneCaptureComponent->ShowFlags.SetTemporalAA(true); + + RenderTarget = NewObject(this, UTextureRenderTarget2D::StaticClass()); - // Set capture source and image properties based on camera type if (CameraType == EROS2CameraType::RGB) { - SceneCaptureComponent->CaptureSource = ESceneCaptureSource::SCS_FinalColorHDR; - RenderTarget = NewObject(this, UTextureRenderTarget2D::StaticClass()); + SceneCaptureComponent->CaptureSource = ESceneCaptureSource::SCS_SceneColorHDR; + + RenderTarget->RenderTargetFormat = ETextureRenderTargetFormat::RTF_RGBA8; RenderTarget->InitCustomFormat(Width, Height, EPixelFormat::PF_B8G8R8A8, true); Data.Encoding = TEXT("bgr8"); Data.Step = Width * 3; + Data.Data.AddUninitialized(Width * Height * 3); } else if (CameraType == EROS2CameraType::DEPTH) { CameraComponent->PostProcessSettings.WeightedBlendables.Array.Add( - FWeightedBlendable(1.0f, GetBufferVisualizationData().GetMaterial(TEXT("SceneDepth")))); + FWeightedBlendable(1.0f, GetBufferVisualizationData().GetMaterial(TEXT("SceneDepth"))) + ); SceneCaptureComponent->PostProcessSettings = CameraComponent->PostProcessSettings; SceneCaptureComponent->CaptureSource = ESceneCaptureSource::SCS_SceneDepth; - RenderTarget = NewObject(this, UTextureRenderTarget2D::StaticClass()); + RenderTarget->RenderTargetFormat = ETextureRenderTargetFormat::RTF_RGBA32f; RenderTarget->InitCustomFormat(Width, Height, EPixelFormat::PF_FloatRGBA, false); + Data.Encoding = TEXT("32FC1"); Data.Step = Width * 4; + Data.Data.AddUninitialized(Width * Height * 4); + } + else if (CameraType == EROS2CameraType::SEGMENT) + { + SceneCaptureComponent->CaptureSource = ESceneCaptureSource::SCS_FinalColorLDR; + SceneCaptureComponent->ShowFlags.SetPostProcessing(true); + + // material reference: /Script/Engine.Material'/RapyutaSimulationPlugins/Materials/PP_Segmentation.PP_Segmentation' + UMaterial* PostProcessMaterial = LoadObject(nullptr, TEXT("/RapyutaSimulationPlugins/Materials/PP_Segmentation.PP_Segmentation")); + + if(PostProcessMaterial){ // check nullptr + SceneCaptureComponent->AddOrUpdateBlendable(PostProcessMaterial); + } else { + UE_LOG(LogTemp, Log, TEXT("No PostProcessMaterial is assigend")); + } + + RenderTarget->RenderTargetFormat = ETextureRenderTargetFormat::RTF_RGBA8; + RenderTarget->InitCustomFormat(Width, Height, EPixelFormat::PF_B8G8R8A8, true); + + Data.Encoding = TEXT("bgr8"); + Data.Step = Width * 3; + Data.Data.AddUninitialized(Width * Height * 3); } // Common setup @@ -50,15 +77,13 @@ void URRROS2CameraComponent::PreInitializePublisher(UROS2NodeComponent* InROS2No Data.Header.FrameId = FrameId; Data.Width = Width; Data.Height = Height; - Data.Data.AddUninitialized(Width * Height * (CameraType == EROS2CameraType::RGB ? 3 : 4)); QueueSize = QueueSize < 1 ? 1 : QueueSize; // QueueSize should be more than 1 Super::PreInitializePublisher(InROS2Node, InTopicName); } void URRROS2CameraComponent::SensorUpdate() { - if (Render) - { + if (Render) { SceneCaptureComponent->CaptureScene(); CaptureNonBlocking(); } @@ -94,7 +119,7 @@ void URRROS2CameraComponent::CaptureNonBlocking() FIntRect rect(0, 0, renderTargetResource->GetSizeXY().X, renderTargetResource->GetSizeXY().Y); FReadSurfaceDataFlags flags(RCM_UNorm, CubeFace_MAX); - if (CameraType == EROS2CameraType::RGB) + if (CameraType == EROS2CameraType::RGB || CameraType == EROS2CameraType::SEGMENT) { FReadSurfaceContextRGB readSurfaceContext = {renderTargetResource, &(renderRequest->Image), rect, flags}; @@ -102,10 +127,11 @@ void URRROS2CameraComponent::CaptureNonBlocking() ( [readSurfaceContext, this](FRHICommandListImmediate& RHICmdList) { - RHICmdList.ReadSurfaceData(readSurfaceContext.SrcRenderTarget->GetRenderTargetTexture(), - readSurfaceContext.Rect, - *readSurfaceContext.OutData, - readSurfaceContext.Flags); + RHICmdList.ReadSurfaceData( + readSurfaceContext.SrcRenderTarget->GetRenderTargetTexture(), + readSurfaceContext.Rect, + *readSurfaceContext.OutData, + readSurfaceContext.Flags); }); } else if (CameraType == EROS2CameraType::DEPTH) @@ -116,10 +142,11 @@ void URRROS2CameraComponent::CaptureNonBlocking() ( [readSurfaceContext, this](FRHICommandListImmediate& RHICmdList) { - RHICmdList.ReadSurfaceFloatData(readSurfaceContext.SrcRenderTarget->GetRenderTargetTexture(), - readSurfaceContext.Rect, - *readSurfaceContext.Depth, - readSurfaceContext.Flags); + RHICmdList.ReadSurfaceFloatData( + readSurfaceContext.SrcRenderTarget->GetRenderTargetTexture(), + readSurfaceContext.Rect, + *readSurfaceContext.Depth, + readSurfaceContext.Flags); }); } @@ -140,8 +167,7 @@ void URRROS2CameraComponent::CaptureNonBlocking() FROSImg URRROS2CameraComponent::GetROS2Data() { - if (!RenderRequestQueue.IsEmpty()) - { + if (!RenderRequestQueue.IsEmpty()) { // Timestamp Data.Header.Stamp = URRConversionUtils::FloatToROSStamp(UGameplayStatics::GetTimeSeconds(GetWorld())); // Peek the next RenderRequest from queue @@ -149,17 +175,17 @@ FROSImg URRROS2CameraComponent::GetROS2Data() RenderRequestQueue.Peek(nextRenderRequest); if (nextRenderRequest && nextRenderRequest->RenderFence.IsFenceComplete()) { - if (CameraType == EROS2CameraType::RGB) + if (CameraType == EROS2CameraType::RGB || CameraType == EROS2CameraType::SEGMENT) { // Process RGB data - for (int i = 0; i < nextRenderRequest->Image.Num(); i++) + for (int i = 0; i < nextRenderRequest->Image.Num(); i++) { Data.Data[i * 3 + 0] = nextRenderRequest->Image[i].B; Data.Data[i * 3 + 1] = nextRenderRequest->Image[i].G; Data.Data[i * 3 + 2] = nextRenderRequest->Image[i].R; } } - else if (CameraType == EROS2CameraType::DEPTH) + else if (CameraType == EROS2CameraType::DEPTH) { // Process Depth data for (int i = 0; i < nextRenderRequest->Depth.Num(); i++) @@ -174,6 +200,7 @@ FROSImg URRROS2CameraComponent::GetROS2Data() QueueCount--; delete nextRenderRequest; } + } return Data; } @@ -181,4 +208,4 @@ FROSImg URRROS2CameraComponent::GetROS2Data() void URRROS2CameraComponent::SetROS2Msg(UROS2GenericMsg* InMessage) { CastChecked(InMessage)->SetMsg(GetROS2Data()); -} +} \ No newline at end of file diff --git a/Source/RapyutaSimulationPlugins/Public/Sensors/RRROS2CameraComponent.h b/Source/RapyutaSimulationPlugins/Public/Sensors/RRROS2CameraComponent.h index 1d49eed1..b9a1f23d 100644 --- a/Source/RapyutaSimulationPlugins/Public/Sensors/RRROS2CameraComponent.h +++ b/Source/RapyutaSimulationPlugins/Public/Sensors/RRROS2CameraComponent.h @@ -38,7 +38,8 @@ UENUM(BlueprintType) enum class EROS2CameraType : uint8 { RGB UMETA(DisplayName = "RGB"), - DEPTH UMETA(DisplayName = "Depth") + DEPTH UMETA(DisplayName = "Depth"), + SEGMENT UMETA(DisplayName = "Segment") }; /** @@ -115,6 +116,7 @@ class RAPYUTASIMULATIONPLUGINS_API URRROS2CameraComponent : public URRROS2BaseSe UPROPERTY(EditAnywhere, BlueprintReadWrite) EROS2CameraType CameraType = EROS2CameraType::RGB; + // Allow to disable rendering (for performance, it still publishes still image) UPROPERTY(EditAnywhere, BlueprintReadWrite) bool Render = true;