From 0133556d2c49273e069f241868093afc440b663d Mon Sep 17 00:00:00 2001 From: Akshay Chawla Date: Fri, 29 Aug 2025 10:48:07 +0100 Subject: [PATCH 1/6] Added a new externalfile server --- Makefile | 3 +- .../utils/grpc_management_plane_utils.go | 2 +- test/mock/grpc/README.md | 3 ++ test/mock/grpc/cmd/main.go | 38 ++++++++++++++++--- .../grpc/mock_management_command_service.go | 28 +++++++++++++- test/mock/grpc/mock_management_server.go | 10 +++-- 6 files changed, 72 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index d436d3582..7b6c34264 100644 --- a/Makefile +++ b/Makefile @@ -66,6 +66,7 @@ DEB_PACKAGE := ./build/$(PACKAGE_NAME).deb RPM_PACKAGE := ./build/$(PACKAGE_NAME).rpm MOCK_MANAGEMENT_PLANE_CONFIG_DIRECTORY ?= +MOCK_MANAGEMENT_PLANE_EXTERNAL_FILE_SERVER ?= MOCK_MANAGEMENT_PLANE_LOG_LEVEL ?= INFO MOCK_MANAGEMENT_PLANE_GRPC_ADDRESS ?= 127.0.0.1:0 MOCK_MANAGEMENT_PLANE_API_ADDRESS ?= 127.0.0.1:0 @@ -193,7 +194,7 @@ race-condition-dev: ## Run agent executable with race condition detection run-mock-management-grpc-server: ## Run mock management plane gRPC server @echo "🖲️ Running mock management plane gRPC server" - $(GORUN) test/mock/grpc/cmd/main.go -configDirectory=$(MOCK_MANAGEMENT_PLANE_CONFIG_DIRECTORY) -logLevel=$(MOCK_MANAGEMENT_PLANE_LOG_LEVEL) -grpcAddress=$(MOCK_MANAGEMENT_PLANE_GRPC_ADDRESS) -apiAddress=$(MOCK_MANAGEMENT_PLANE_API_ADDRESS) + $(GORUN) test/mock/grpc/cmd/main.go -configDirectory=$(MOCK_MANAGEMENT_PLANE_CONFIG_DIRECTORY) -logLevel=$(MOCK_MANAGEMENT_PLANE_LOG_LEVEL) -grpcAddress=$(MOCK_MANAGEMENT_PLANE_GRPC_ADDRESS) -apiAddress=$(MOCK_MANAGEMENT_PLANE_API_ADDRESS) -externalFileServer=$(MOCK_MANAGEMENT_PLANE_EXTERNAL_FILE_SERVER) .PHONY: build-test-nginx-plus-and-nap-image diff --git a/test/integration/utils/grpc_management_plane_utils.go b/test/integration/utils/grpc_management_plane_utils.go index de8d20f72..ad11b6dce 100644 --- a/test/integration/utils/grpc_management_plane_utils.go +++ b/test/integration/utils/grpc_management_plane_utils.go @@ -188,7 +188,7 @@ func setupLocalEnvironment(tb testing.TB) { ctx := context.Background() requestChan := make(chan *mpi.ManagementPlaneRequest) - server := mockGrpc.NewCommandService(requestChan, os.TempDir()) + server := mockGrpc.NewCommandService(requestChan, os.TempDir(), os.TempDir()) go func(tb testing.TB) { tb.Helper() diff --git a/test/mock/grpc/README.md b/test/mock/grpc/README.md index 7272f5457..cb10c1c92 100644 --- a/test/mock/grpc/README.md +++ b/test/mock/grpc/README.md @@ -12,6 +12,7 @@ To override these behaviours the following environment variables can be set to o MOCK_MANAGEMENT_PLANE_GRPC_ADDRESS=127.0.0.1:9091 MOCK_MANAGEMENT_PLANE_API_ADDRESS=127.0.0.1:9092 MOCK_MANAGEMENT_PLANE_CONFIG_DIRECTORY=/tmp/ +MOCK_MANAGEMENT_PLANE_EXTERNAL_FILE_SERVER=/tmp/ ``` Before starting the NGINX Agent, update the agent configuration with the following command config block @@ -35,6 +36,8 @@ GET http://127.0.0.1:9092/api/v1/responses POST http://127.0.0.1:9092/api/v1/requests POST http://127.0.0.1:9092/api/v1/instance//config/apply + +GET http://127.0.0.1:9092/api/v1/externalfile/ ``` # Endpoints diff --git a/test/mock/grpc/cmd/main.go b/test/mock/grpc/cmd/main.go index 250e476fa..a50f034ac 100644 --- a/test/mock/grpc/cmd/main.go +++ b/test/mock/grpc/cmd/main.go @@ -27,11 +27,12 @@ const ( ) var ( - sleepDuration = flag.Duration("sleepDuration", defaultSleepDuration, "duration between changes in health") - configDirectory = flag.String("configDirectory", "", "set the directory where the config files are stored") - grpcAddress = flag.String("grpcAddress", "127.0.0.1:0", "set the gRPC address to run the server on") - apiAddress = flag.String("apiAddress", "127.0.0.1:0", "set the API address to run the server on") - logLevel = flag.String("logLevel", "INFO", "set the log level") + sleepDuration = flag.Duration("sleepDuration", defaultSleepDuration, "duration between changes in health") + configDirectory = flag.String("configDirectory", "", "set the directory where the config files are stored") + externalFileServer = flag.String("externalFileServer", "", "set the directory for external file server") + grpcAddress = flag.String("grpcAddress", "127.0.0.1:0", "set the gRPC address to run the server on") + apiAddress = flag.String("apiAddress", "127.0.0.1:0", "set the API address to run the server on") + logLevel = flag.String("logLevel", "INFO", "set the log level") ) func main() { @@ -74,9 +75,18 @@ func main() { } } + if externalFileServer == nil || *externalFileServer == "" { + defaultExternalFileServer, externalFileServerErr := generateDefaultExternalFileSevrevDirectory() + externalFileServer = &defaultExternalFileServer + if externalFileServerErr != nil { + slog.ErrorContext(ctx, "Failed to create default config directory", "error", err) + os.Exit(1) + } + } + slog.DebugContext(ctx, "Config directory", "directory", *configDirectory) - _, err = grpc.NewMockManagementServer(ctx, *apiAddress, agentConfig, configDirectory) + _, err = grpc.NewMockManagementServer(ctx, *apiAddress, agentConfig, configDirectory, externalFileServer) if err != nil { slog.ErrorContext(ctx, "Failed to start mock management server", "error", err) os.Exit(1) @@ -99,3 +109,19 @@ func generateDefaultConfigDirectory() (string, error) { return configDirectory, nil } + +func generateDefaultExternalFileSevrevDirectory() (string, error) { + slog.Info("Generating external file server directory") + tempDirectory := os.TempDir() + externalFileServer := filepath.Join(tempDirectory, "externalfileserver") + + err := os.MkdirAll(externalFileServer, directoryPermissions) + if err != nil { + slog.Error("Failed to create external file server directory", "error", err) + return "", err + } + + slog.Info("Created default external file server directory", "directory", externalFileServer) + + return externalFileServer, nil +} diff --git a/test/mock/grpc/mock_management_command_service.go b/test/mock/grpc/mock_management_command_service.go index ff0d753af..5f2115405 100644 --- a/test/mock/grpc/mock_management_command_service.go +++ b/test/mock/grpc/mock_management_command_service.go @@ -39,6 +39,7 @@ type CommandService struct { updateDataPlaneHealthRequest *mpi.UpdateDataPlaneHealthRequest instanceFiles map[string][]*mpi.File // key is instanceID configDirectory string + externalFileServer string dataPlaneResponses []*mpi.DataPlaneResponse updateDataPlaneHealthMutex sync.Mutex connectionMutex sync.Mutex @@ -50,7 +51,10 @@ func init() { gin.SetMode(gin.ReleaseMode) } -func NewCommandService(requestChan chan *mpi.ManagementPlaneRequest, configDirectory string) *CommandService { +func NewCommandService( + requestChan chan *mpi.ManagementPlaneRequest, + configDirectory string, + externalFileServer string) *CommandService { cs := &CommandService{ requestChan: requestChan, connectionMutex: sync.Mutex{}, @@ -58,6 +62,7 @@ func NewCommandService(requestChan chan *mpi.ManagementPlaneRequest, configDirec updateDataPlaneHealthMutex: sync.Mutex{}, dataPlaneResponsesMutex: sync.Mutex{}, configDirectory: configDirectory, + externalFileServer: externalFileServer, instanceFiles: make(map[string][]*mpi.File), } @@ -246,6 +251,7 @@ func (cs *CommandService) createServer(logger *slog.Logger) { cs.addResponseAndRequestEndpoints() cs.addConfigApplyEndpoint() cs.addConfigEndpoint() + cs.addExternalFileServerEndpoint() } func (cs *CommandService) addConnectionEndpoint() { @@ -410,6 +416,26 @@ func (cs *CommandService) addConfigEndpoint() { }) } +func (cs *CommandService) addExternalFileServerEndpoint() { + // This API will serve individual files from the external directory + cs.server.GET("/api/v1/externalfile/:filename", func(c *gin.Context) { + filename := c.Param("filename") + filePath := filepath.Join(cs.externalFileServer, filename) + + // Check if the file exists + if _, err := os.Stat(filePath); os.IsNotExist(err) { + c.JSON(http.StatusNotFound, gin.H{"error": "File not found"}) + + return + } + + // Serve the file + c.File(filePath) + }) + + slog.Info("Serving individual external files from", "directory", cs.externalFileServer) +} + func (cs *CommandService) findInstanceConfigFiles(instanceID string) (configFiles []*mpi.File, err error) { instanceDirectory := filepath.Join(cs.configDirectory, instanceID) diff --git a/test/mock/grpc/mock_management_server.go b/test/mock/grpc/mock_management_server.go index 155c267fe..233c2b18e 100644 --- a/test/mock/grpc/mock_management_server.go +++ b/test/mock/grpc/mock_management_server.go @@ -18,7 +18,7 @@ import ( "syscall" "time" - "github.com/nginx/agent/v3/api/grpc/mpi/v1" + v1 "github.com/nginx/agent/v3/api/grpc/mpi/v1" "github.com/nginx/agent/v3/internal/config" "github.com/bufbuild/protovalidate-go" @@ -68,11 +68,13 @@ func NewMockManagementServer( apiAddress string, agentConfig *config.Config, configDirectory *string, + externalFileServer *string, ) (*MockManagementServer, error) { var err error requestChan := make(chan *v1.ManagementPlaneRequest) - commandService := serveCommandService(ctx, apiAddress, agentConfig, requestChan, *configDirectory) + commandService := serveCommandService(ctx, apiAddress, agentConfig, requestChan, *configDirectory, + *externalFileServer) var fileServer *FileService @@ -180,14 +182,16 @@ func serverOptions(agentConfig *config.Config) []grpc.ServerOption { return opts } +//nolint:revive // Have to add a new parameter here to support external file server func serveCommandService( ctx context.Context, apiAddress string, agentConfig *config.Config, requestChan chan *v1.ManagementPlaneRequest, configDirectory string, + externalFileServer string, ) *CommandService { - commandServer := NewCommandService(requestChan, configDirectory) + commandServer := NewCommandService(requestChan, configDirectory, externalFileServer) go func() { cmdListener, listenerErr := createListener(ctx, apiAddress, agentConfig) From d9076e30fa9ba264d2494a7a5cb39dc93f81ba0d Mon Sep 17 00:00:00 2001 From: Akshay Chawla Date: Fri, 29 Aug 2025 14:49:22 +0100 Subject: [PATCH 2/6] Updated config-apply api --- api/grpc/mpi/v1/files.pb.go | 245 +++++++++++------- api/grpc/mpi/v1/files.pb.validate.go | 137 ++++++++++ api/grpc/mpi/v1/files.proto | 7 + docs/proto/protos.md | 17 ++ test/mock/grpc/README.md | 71 ++++- .../grpc/mock_management_command_service.go | 55 +++- test/mock/grpc/mock_management_server.go | 2 +- 7 files changed, 438 insertions(+), 96 deletions(-) diff --git a/api/grpc/mpi/v1/files.pb.go b/api/grpc/mpi/v1/files.pb.go index 1873a19e2..4fe57bb7c 100644 --- a/api/grpc/mpi/v1/files.pb.go +++ b/api/grpc/mpi/v1/files.pb.go @@ -688,9 +688,11 @@ type File struct { // Meta information about the file, the name (including path) and hash FileMeta *FileMeta `protobuf:"bytes,1,opt,name=file_meta,json=fileMeta,proto3" json:"file_meta,omitempty"` // Unmanaged files will not be modified - Unmanaged bool `protobuf:"varint,2,opt,name=unmanaged,proto3" json:"unmanaged,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + Unmanaged bool `protobuf:"varint,2,opt,name=unmanaged,proto3" json:"unmanaged,omitempty"` + // external file source + ExternalDataSource *ExternalDataSource `protobuf:"bytes,3,opt,name=external_data_source,json=externalDataSource,proto3,oneof" json:"external_data_source,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *File) Reset() { @@ -737,6 +739,58 @@ func (x *File) GetUnmanaged() bool { return false } +func (x *File) GetExternalDataSource() *ExternalDataSource { + if x != nil { + return x.ExternalDataSource + } + return nil +} + +type ExternalDataSource struct { + state protoimpl.MessageState `protogen:"open.v1"` + // URL to the location of an external file + Location string `protobuf:"bytes,1,opt,name=location,proto3" json:"location,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ExternalDataSource) Reset() { + *x = ExternalDataSource{} + mi := &file_mpi_v1_files_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ExternalDataSource) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ExternalDataSource) ProtoMessage() {} + +func (x *ExternalDataSource) ProtoReflect() protoreflect.Message { + mi := &file_mpi_v1_files_proto_msgTypes[10] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ExternalDataSource.ProtoReflect.Descriptor instead. +func (*ExternalDataSource) Descriptor() ([]byte, []int) { + return file_mpi_v1_files_proto_rawDescGZIP(), []int{10} +} + +func (x *ExternalDataSource) GetLocation() string { + if x != nil { + return x.Location + } + return "" +} + // Represents the get file request type GetFileRequest struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -750,7 +804,7 @@ type GetFileRequest struct { func (x *GetFileRequest) Reset() { *x = GetFileRequest{} - mi := &file_mpi_v1_files_proto_msgTypes[10] + mi := &file_mpi_v1_files_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -762,7 +816,7 @@ func (x *GetFileRequest) String() string { func (*GetFileRequest) ProtoMessage() {} func (x *GetFileRequest) ProtoReflect() protoreflect.Message { - mi := &file_mpi_v1_files_proto_msgTypes[10] + mi := &file_mpi_v1_files_proto_msgTypes[11] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -775,7 +829,7 @@ func (x *GetFileRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetFileRequest.ProtoReflect.Descriptor instead. func (*GetFileRequest) Descriptor() ([]byte, []int) { - return file_mpi_v1_files_proto_rawDescGZIP(), []int{10} + return file_mpi_v1_files_proto_rawDescGZIP(), []int{11} } func (x *GetFileRequest) GetMessageMeta() *MessageMeta { @@ -803,7 +857,7 @@ type GetFileResponse struct { func (x *GetFileResponse) Reset() { *x = GetFileResponse{} - mi := &file_mpi_v1_files_proto_msgTypes[11] + mi := &file_mpi_v1_files_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -815,7 +869,7 @@ func (x *GetFileResponse) String() string { func (*GetFileResponse) ProtoMessage() {} func (x *GetFileResponse) ProtoReflect() protoreflect.Message { - mi := &file_mpi_v1_files_proto_msgTypes[11] + mi := &file_mpi_v1_files_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -828,7 +882,7 @@ func (x *GetFileResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetFileResponse.ProtoReflect.Descriptor instead. func (*GetFileResponse) Descriptor() ([]byte, []int) { - return file_mpi_v1_files_proto_rawDescGZIP(), []int{11} + return file_mpi_v1_files_proto_rawDescGZIP(), []int{12} } func (x *GetFileResponse) GetContents() *FileContents { @@ -849,7 +903,7 @@ type FileContents struct { func (x *FileContents) Reset() { *x = FileContents{} - mi := &file_mpi_v1_files_proto_msgTypes[12] + mi := &file_mpi_v1_files_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -861,7 +915,7 @@ func (x *FileContents) String() string { func (*FileContents) ProtoMessage() {} func (x *FileContents) ProtoReflect() protoreflect.Message { - mi := &file_mpi_v1_files_proto_msgTypes[12] + mi := &file_mpi_v1_files_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -874,7 +928,7 @@ func (x *FileContents) ProtoReflect() protoreflect.Message { // Deprecated: Use FileContents.ProtoReflect.Descriptor instead. func (*FileContents) Descriptor() ([]byte, []int) { - return file_mpi_v1_files_proto_rawDescGZIP(), []int{12} + return file_mpi_v1_files_proto_rawDescGZIP(), []int{13} } func (x *FileContents) GetContents() []byte { @@ -909,7 +963,7 @@ type FileMeta struct { func (x *FileMeta) Reset() { *x = FileMeta{} - mi := &file_mpi_v1_files_proto_msgTypes[13] + mi := &file_mpi_v1_files_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -921,7 +975,7 @@ func (x *FileMeta) String() string { func (*FileMeta) ProtoMessage() {} func (x *FileMeta) ProtoReflect() protoreflect.Message { - mi := &file_mpi_v1_files_proto_msgTypes[13] + mi := &file_mpi_v1_files_proto_msgTypes[14] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -934,7 +988,7 @@ func (x *FileMeta) ProtoReflect() protoreflect.Message { // Deprecated: Use FileMeta.ProtoReflect.Descriptor instead. func (*FileMeta) Descriptor() ([]byte, []int) { - return file_mpi_v1_files_proto_rawDescGZIP(), []int{13} + return file_mpi_v1_files_proto_rawDescGZIP(), []int{14} } func (x *FileMeta) GetName() string { @@ -1013,7 +1067,7 @@ type UpdateFileRequest struct { func (x *UpdateFileRequest) Reset() { *x = UpdateFileRequest{} - mi := &file_mpi_v1_files_proto_msgTypes[14] + mi := &file_mpi_v1_files_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1025,7 +1079,7 @@ func (x *UpdateFileRequest) String() string { func (*UpdateFileRequest) ProtoMessage() {} func (x *UpdateFileRequest) ProtoReflect() protoreflect.Message { - mi := &file_mpi_v1_files_proto_msgTypes[14] + mi := &file_mpi_v1_files_proto_msgTypes[15] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1038,7 +1092,7 @@ func (x *UpdateFileRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use UpdateFileRequest.ProtoReflect.Descriptor instead. func (*UpdateFileRequest) Descriptor() ([]byte, []int) { - return file_mpi_v1_files_proto_rawDescGZIP(), []int{14} + return file_mpi_v1_files_proto_rawDescGZIP(), []int{15} } func (x *UpdateFileRequest) GetFile() *File { @@ -1073,7 +1127,7 @@ type UpdateFileResponse struct { func (x *UpdateFileResponse) Reset() { *x = UpdateFileResponse{} - mi := &file_mpi_v1_files_proto_msgTypes[15] + mi := &file_mpi_v1_files_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1085,7 +1139,7 @@ func (x *UpdateFileResponse) String() string { func (*UpdateFileResponse) ProtoMessage() {} func (x *UpdateFileResponse) ProtoReflect() protoreflect.Message { - mi := &file_mpi_v1_files_proto_msgTypes[15] + mi := &file_mpi_v1_files_proto_msgTypes[16] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1098,7 +1152,7 @@ func (x *UpdateFileResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use UpdateFileResponse.ProtoReflect.Descriptor instead. func (*UpdateFileResponse) Descriptor() ([]byte, []int) { - return file_mpi_v1_files_proto_rawDescGZIP(), []int{15} + return file_mpi_v1_files_proto_rawDescGZIP(), []int{16} } func (x *UpdateFileResponse) GetFileMeta() *FileMeta { @@ -1132,7 +1186,7 @@ type CertificateMeta struct { func (x *CertificateMeta) Reset() { *x = CertificateMeta{} - mi := &file_mpi_v1_files_proto_msgTypes[16] + mi := &file_mpi_v1_files_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1144,7 +1198,7 @@ func (x *CertificateMeta) String() string { func (*CertificateMeta) ProtoMessage() {} func (x *CertificateMeta) ProtoReflect() protoreflect.Message { - mi := &file_mpi_v1_files_proto_msgTypes[16] + mi := &file_mpi_v1_files_proto_msgTypes[17] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1157,7 +1211,7 @@ func (x *CertificateMeta) ProtoReflect() protoreflect.Message { // Deprecated: Use CertificateMeta.ProtoReflect.Descriptor instead. func (*CertificateMeta) Descriptor() ([]byte, []int) { - return file_mpi_v1_files_proto_rawDescGZIP(), []int{16} + return file_mpi_v1_files_proto_rawDescGZIP(), []int{17} } func (x *CertificateMeta) GetSerialNumber() string { @@ -1222,7 +1276,7 @@ type CertificateDates struct { func (x *CertificateDates) Reset() { *x = CertificateDates{} - mi := &file_mpi_v1_files_proto_msgTypes[17] + mi := &file_mpi_v1_files_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1234,7 +1288,7 @@ func (x *CertificateDates) String() string { func (*CertificateDates) ProtoMessage() {} func (x *CertificateDates) ProtoReflect() protoreflect.Message { - mi := &file_mpi_v1_files_proto_msgTypes[17] + mi := &file_mpi_v1_files_proto_msgTypes[18] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1247,7 +1301,7 @@ func (x *CertificateDates) ProtoReflect() protoreflect.Message { // Deprecated: Use CertificateDates.ProtoReflect.Descriptor instead. func (*CertificateDates) Descriptor() ([]byte, []int) { - return file_mpi_v1_files_proto_rawDescGZIP(), []int{17} + return file_mpi_v1_files_proto_rawDescGZIP(), []int{18} } func (x *CertificateDates) GetNotBefore() int64 { @@ -1277,7 +1331,7 @@ type SubjectAlternativeNames struct { func (x *SubjectAlternativeNames) Reset() { *x = SubjectAlternativeNames{} - mi := &file_mpi_v1_files_proto_msgTypes[18] + mi := &file_mpi_v1_files_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1289,7 +1343,7 @@ func (x *SubjectAlternativeNames) String() string { func (*SubjectAlternativeNames) ProtoMessage() {} func (x *SubjectAlternativeNames) ProtoReflect() protoreflect.Message { - mi := &file_mpi_v1_files_proto_msgTypes[18] + mi := &file_mpi_v1_files_proto_msgTypes[19] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1302,7 +1356,7 @@ func (x *SubjectAlternativeNames) ProtoReflect() protoreflect.Message { // Deprecated: Use SubjectAlternativeNames.ProtoReflect.Descriptor instead. func (*SubjectAlternativeNames) Descriptor() ([]byte, []int) { - return file_mpi_v1_files_proto_rawDescGZIP(), []int{18} + return file_mpi_v1_files_proto_rawDescGZIP(), []int{19} } func (x *SubjectAlternativeNames) GetDnsNames() []string { @@ -1354,7 +1408,7 @@ type X509Name struct { func (x *X509Name) Reset() { *x = X509Name{} - mi := &file_mpi_v1_files_proto_msgTypes[19] + mi := &file_mpi_v1_files_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1366,7 +1420,7 @@ func (x *X509Name) String() string { func (*X509Name) ProtoMessage() {} func (x *X509Name) ProtoReflect() protoreflect.Message { - mi := &file_mpi_v1_files_proto_msgTypes[19] + mi := &file_mpi_v1_files_proto_msgTypes[20] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1379,7 +1433,7 @@ func (x *X509Name) ProtoReflect() protoreflect.Message { // Deprecated: Use X509Name.ProtoReflect.Descriptor instead. func (*X509Name) Descriptor() ([]byte, []int) { - return file_mpi_v1_files_proto_rawDescGZIP(), []int{19} + return file_mpi_v1_files_proto_rawDescGZIP(), []int{20} } func (x *X509Name) GetCountry() []string { @@ -1471,7 +1525,7 @@ type AttributeTypeAndValue struct { func (x *AttributeTypeAndValue) Reset() { *x = AttributeTypeAndValue{} - mi := &file_mpi_v1_files_proto_msgTypes[20] + mi := &file_mpi_v1_files_proto_msgTypes[21] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1483,7 +1537,7 @@ func (x *AttributeTypeAndValue) String() string { func (*AttributeTypeAndValue) ProtoMessage() {} func (x *AttributeTypeAndValue) ProtoReflect() protoreflect.Message { - mi := &file_mpi_v1_files_proto_msgTypes[20] + mi := &file_mpi_v1_files_proto_msgTypes[21] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1496,7 +1550,7 @@ func (x *AttributeTypeAndValue) ProtoReflect() protoreflect.Message { // Deprecated: Use AttributeTypeAndValue.ProtoReflect.Descriptor instead. func (*AttributeTypeAndValue) Descriptor() ([]byte, []int) { - return file_mpi_v1_files_proto_rawDescGZIP(), []int{20} + return file_mpi_v1_files_proto_rawDescGZIP(), []int{21} } func (x *AttributeTypeAndValue) GetType() string { @@ -1549,10 +1603,14 @@ const file_mpi_v1_files_proto_rawDesc = "" + "\x05files\x18\x01 \x03(\v2\f.mpi.v1.FileR\x05files\x12<\n" + "\x0econfig_version\x18\x02 \x01(\v2\x15.mpi.v1.ConfigVersionR\rconfigVersion\x12\x1f\n" + "\vconfig_path\x18\x03 \x01(\tR\n" + - "configPath\"S\n" + + "configPath\"\xbf\x01\n" + "\x04File\x12-\n" + "\tfile_meta\x18\x01 \x01(\v2\x10.mpi.v1.FileMetaR\bfileMeta\x12\x1c\n" + - "\tunmanaged\x18\x02 \x01(\bR\tunmanaged\"w\n" + + "\tunmanaged\x18\x02 \x01(\bR\tunmanaged\x12Q\n" + + "\x14external_data_source\x18\x03 \x01(\v2\x1a.mpi.v1.ExternalDataSourceH\x00R\x12externalDataSource\x88\x01\x01B\x17\n" + + "\x15_external_data_source\"0\n" + + "\x12ExternalDataSource\x12\x1a\n" + + "\blocation\x18\x01 \x01(\tR\blocation\"w\n" + "\x0eGetFileRequest\x126\n" + "\fmessage_meta\x18\x01 \x01(\v2\x13.mpi.v1.MessageMetaR\vmessageMeta\x12-\n" + "\tfile_meta\x18\x02 \x01(\v2\x10.mpi.v1.FileMetaR\bfileMeta\"C\n" + @@ -1649,7 +1707,7 @@ func file_mpi_v1_files_proto_rawDescGZIP() []byte { } var file_mpi_v1_files_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_mpi_v1_files_proto_msgTypes = make([]protoimpl.MessageInfo, 21) +var file_mpi_v1_files_proto_msgTypes = make([]protoimpl.MessageInfo, 22) var file_mpi_v1_files_proto_goTypes = []any{ (SignatureAlgorithm)(0), // 0: mpi.v1.SignatureAlgorithm (*FileDataChunk)(nil), // 1: mpi.v1.FileDataChunk @@ -1662,67 +1720,69 @@ var file_mpi_v1_files_proto_goTypes = []any{ (*ConfigVersion)(nil), // 8: mpi.v1.ConfigVersion (*FileOverview)(nil), // 9: mpi.v1.FileOverview (*File)(nil), // 10: mpi.v1.File - (*GetFileRequest)(nil), // 11: mpi.v1.GetFileRequest - (*GetFileResponse)(nil), // 12: mpi.v1.GetFileResponse - (*FileContents)(nil), // 13: mpi.v1.FileContents - (*FileMeta)(nil), // 14: mpi.v1.FileMeta - (*UpdateFileRequest)(nil), // 15: mpi.v1.UpdateFileRequest - (*UpdateFileResponse)(nil), // 16: mpi.v1.UpdateFileResponse - (*CertificateMeta)(nil), // 17: mpi.v1.CertificateMeta - (*CertificateDates)(nil), // 18: mpi.v1.CertificateDates - (*SubjectAlternativeNames)(nil), // 19: mpi.v1.SubjectAlternativeNames - (*X509Name)(nil), // 20: mpi.v1.X509Name - (*AttributeTypeAndValue)(nil), // 21: mpi.v1.AttributeTypeAndValue - (*MessageMeta)(nil), // 22: mpi.v1.MessageMeta - (*timestamppb.Timestamp)(nil), // 23: google.protobuf.Timestamp + (*ExternalDataSource)(nil), // 11: mpi.v1.ExternalDataSource + (*GetFileRequest)(nil), // 12: mpi.v1.GetFileRequest + (*GetFileResponse)(nil), // 13: mpi.v1.GetFileResponse + (*FileContents)(nil), // 14: mpi.v1.FileContents + (*FileMeta)(nil), // 15: mpi.v1.FileMeta + (*UpdateFileRequest)(nil), // 16: mpi.v1.UpdateFileRequest + (*UpdateFileResponse)(nil), // 17: mpi.v1.UpdateFileResponse + (*CertificateMeta)(nil), // 18: mpi.v1.CertificateMeta + (*CertificateDates)(nil), // 19: mpi.v1.CertificateDates + (*SubjectAlternativeNames)(nil), // 20: mpi.v1.SubjectAlternativeNames + (*X509Name)(nil), // 21: mpi.v1.X509Name + (*AttributeTypeAndValue)(nil), // 22: mpi.v1.AttributeTypeAndValue + (*MessageMeta)(nil), // 23: mpi.v1.MessageMeta + (*timestamppb.Timestamp)(nil), // 24: google.protobuf.Timestamp } var file_mpi_v1_files_proto_depIdxs = []int32{ - 22, // 0: mpi.v1.FileDataChunk.meta:type_name -> mpi.v1.MessageMeta + 23, // 0: mpi.v1.FileDataChunk.meta:type_name -> mpi.v1.MessageMeta 2, // 1: mpi.v1.FileDataChunk.header:type_name -> mpi.v1.FileDataChunkHeader 3, // 2: mpi.v1.FileDataChunk.content:type_name -> mpi.v1.FileDataChunkContent - 14, // 3: mpi.v1.FileDataChunkHeader.file_meta:type_name -> mpi.v1.FileMeta - 22, // 4: mpi.v1.GetOverviewRequest.message_meta:type_name -> mpi.v1.MessageMeta + 15, // 3: mpi.v1.FileDataChunkHeader.file_meta:type_name -> mpi.v1.FileMeta + 23, // 4: mpi.v1.GetOverviewRequest.message_meta:type_name -> mpi.v1.MessageMeta 8, // 5: mpi.v1.GetOverviewRequest.config_version:type_name -> mpi.v1.ConfigVersion 9, // 6: mpi.v1.GetOverviewResponse.overview:type_name -> mpi.v1.FileOverview - 22, // 7: mpi.v1.UpdateOverviewRequest.message_meta:type_name -> mpi.v1.MessageMeta + 23, // 7: mpi.v1.UpdateOverviewRequest.message_meta:type_name -> mpi.v1.MessageMeta 9, // 8: mpi.v1.UpdateOverviewRequest.overview:type_name -> mpi.v1.FileOverview 9, // 9: mpi.v1.UpdateOverviewResponse.overview:type_name -> mpi.v1.FileOverview 10, // 10: mpi.v1.FileOverview.files:type_name -> mpi.v1.File 8, // 11: mpi.v1.FileOverview.config_version:type_name -> mpi.v1.ConfigVersion - 14, // 12: mpi.v1.File.file_meta:type_name -> mpi.v1.FileMeta - 22, // 13: mpi.v1.GetFileRequest.message_meta:type_name -> mpi.v1.MessageMeta - 14, // 14: mpi.v1.GetFileRequest.file_meta:type_name -> mpi.v1.FileMeta - 13, // 15: mpi.v1.GetFileResponse.contents:type_name -> mpi.v1.FileContents - 23, // 16: mpi.v1.FileMeta.modified_time:type_name -> google.protobuf.Timestamp - 17, // 17: mpi.v1.FileMeta.certificate_meta:type_name -> mpi.v1.CertificateMeta - 10, // 18: mpi.v1.UpdateFileRequest.file:type_name -> mpi.v1.File - 13, // 19: mpi.v1.UpdateFileRequest.contents:type_name -> mpi.v1.FileContents - 22, // 20: mpi.v1.UpdateFileRequest.message_meta:type_name -> mpi.v1.MessageMeta - 14, // 21: mpi.v1.UpdateFileResponse.file_meta:type_name -> mpi.v1.FileMeta - 20, // 22: mpi.v1.CertificateMeta.issuer:type_name -> mpi.v1.X509Name - 20, // 23: mpi.v1.CertificateMeta.subject:type_name -> mpi.v1.X509Name - 19, // 24: mpi.v1.CertificateMeta.sans:type_name -> mpi.v1.SubjectAlternativeNames - 18, // 25: mpi.v1.CertificateMeta.dates:type_name -> mpi.v1.CertificateDates - 0, // 26: mpi.v1.CertificateMeta.signature_algorithm:type_name -> mpi.v1.SignatureAlgorithm - 21, // 27: mpi.v1.X509Name.names:type_name -> mpi.v1.AttributeTypeAndValue - 21, // 28: mpi.v1.X509Name.extra_names:type_name -> mpi.v1.AttributeTypeAndValue - 4, // 29: mpi.v1.FileService.GetOverview:input_type -> mpi.v1.GetOverviewRequest - 6, // 30: mpi.v1.FileService.UpdateOverview:input_type -> mpi.v1.UpdateOverviewRequest - 11, // 31: mpi.v1.FileService.GetFile:input_type -> mpi.v1.GetFileRequest - 15, // 32: mpi.v1.FileService.UpdateFile:input_type -> mpi.v1.UpdateFileRequest - 11, // 33: mpi.v1.FileService.GetFileStream:input_type -> mpi.v1.GetFileRequest - 1, // 34: mpi.v1.FileService.UpdateFileStream:input_type -> mpi.v1.FileDataChunk - 5, // 35: mpi.v1.FileService.GetOverview:output_type -> mpi.v1.GetOverviewResponse - 7, // 36: mpi.v1.FileService.UpdateOverview:output_type -> mpi.v1.UpdateOverviewResponse - 12, // 37: mpi.v1.FileService.GetFile:output_type -> mpi.v1.GetFileResponse - 16, // 38: mpi.v1.FileService.UpdateFile:output_type -> mpi.v1.UpdateFileResponse - 1, // 39: mpi.v1.FileService.GetFileStream:output_type -> mpi.v1.FileDataChunk - 16, // 40: mpi.v1.FileService.UpdateFileStream:output_type -> mpi.v1.UpdateFileResponse - 35, // [35:41] is the sub-list for method output_type - 29, // [29:35] is the sub-list for method input_type - 29, // [29:29] is the sub-list for extension type_name - 29, // [29:29] is the sub-list for extension extendee - 0, // [0:29] is the sub-list for field type_name + 15, // 12: mpi.v1.File.file_meta:type_name -> mpi.v1.FileMeta + 11, // 13: mpi.v1.File.external_data_source:type_name -> mpi.v1.ExternalDataSource + 23, // 14: mpi.v1.GetFileRequest.message_meta:type_name -> mpi.v1.MessageMeta + 15, // 15: mpi.v1.GetFileRequest.file_meta:type_name -> mpi.v1.FileMeta + 14, // 16: mpi.v1.GetFileResponse.contents:type_name -> mpi.v1.FileContents + 24, // 17: mpi.v1.FileMeta.modified_time:type_name -> google.protobuf.Timestamp + 18, // 18: mpi.v1.FileMeta.certificate_meta:type_name -> mpi.v1.CertificateMeta + 10, // 19: mpi.v1.UpdateFileRequest.file:type_name -> mpi.v1.File + 14, // 20: mpi.v1.UpdateFileRequest.contents:type_name -> mpi.v1.FileContents + 23, // 21: mpi.v1.UpdateFileRequest.message_meta:type_name -> mpi.v1.MessageMeta + 15, // 22: mpi.v1.UpdateFileResponse.file_meta:type_name -> mpi.v1.FileMeta + 21, // 23: mpi.v1.CertificateMeta.issuer:type_name -> mpi.v1.X509Name + 21, // 24: mpi.v1.CertificateMeta.subject:type_name -> mpi.v1.X509Name + 20, // 25: mpi.v1.CertificateMeta.sans:type_name -> mpi.v1.SubjectAlternativeNames + 19, // 26: mpi.v1.CertificateMeta.dates:type_name -> mpi.v1.CertificateDates + 0, // 27: mpi.v1.CertificateMeta.signature_algorithm:type_name -> mpi.v1.SignatureAlgorithm + 22, // 28: mpi.v1.X509Name.names:type_name -> mpi.v1.AttributeTypeAndValue + 22, // 29: mpi.v1.X509Name.extra_names:type_name -> mpi.v1.AttributeTypeAndValue + 4, // 30: mpi.v1.FileService.GetOverview:input_type -> mpi.v1.GetOverviewRequest + 6, // 31: mpi.v1.FileService.UpdateOverview:input_type -> mpi.v1.UpdateOverviewRequest + 12, // 32: mpi.v1.FileService.GetFile:input_type -> mpi.v1.GetFileRequest + 16, // 33: mpi.v1.FileService.UpdateFile:input_type -> mpi.v1.UpdateFileRequest + 12, // 34: mpi.v1.FileService.GetFileStream:input_type -> mpi.v1.GetFileRequest + 1, // 35: mpi.v1.FileService.UpdateFileStream:input_type -> mpi.v1.FileDataChunk + 5, // 36: mpi.v1.FileService.GetOverview:output_type -> mpi.v1.GetOverviewResponse + 7, // 37: mpi.v1.FileService.UpdateOverview:output_type -> mpi.v1.UpdateOverviewResponse + 13, // 38: mpi.v1.FileService.GetFile:output_type -> mpi.v1.GetFileResponse + 17, // 39: mpi.v1.FileService.UpdateFile:output_type -> mpi.v1.UpdateFileResponse + 1, // 40: mpi.v1.FileService.GetFileStream:output_type -> mpi.v1.FileDataChunk + 17, // 41: mpi.v1.FileService.UpdateFileStream:output_type -> mpi.v1.UpdateFileResponse + 36, // [36:42] is the sub-list for method output_type + 30, // [30:36] is the sub-list for method input_type + 30, // [30:30] is the sub-list for extension type_name + 30, // [30:30] is the sub-list for extension extendee + 0, // [0:30] is the sub-list for field type_name } func init() { file_mpi_v1_files_proto_init() } @@ -1735,7 +1795,8 @@ func file_mpi_v1_files_proto_init() { (*FileDataChunk_Header)(nil), (*FileDataChunk_Content)(nil), } - file_mpi_v1_files_proto_msgTypes[13].OneofWrappers = []any{ + file_mpi_v1_files_proto_msgTypes[9].OneofWrappers = []any{} + file_mpi_v1_files_proto_msgTypes[14].OneofWrappers = []any{ (*FileMeta_CertificateMeta)(nil), } type x struct{} @@ -1744,7 +1805,7 @@ func file_mpi_v1_files_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_mpi_v1_files_proto_rawDesc), len(file_mpi_v1_files_proto_rawDesc)), NumEnums: 1, - NumMessages: 21, + NumMessages: 22, NumExtensions: 0, NumServices: 1, }, diff --git a/api/grpc/mpi/v1/files.pb.validate.go b/api/grpc/mpi/v1/files.pb.validate.go index a54f2b32b..12f295922 100644 --- a/api/grpc/mpi/v1/files.pb.validate.go +++ b/api/grpc/mpi/v1/files.pb.validate.go @@ -1394,6 +1394,39 @@ func (m *File) validate(all bool) error { // no validation rules for Unmanaged + if m.ExternalDataSource != nil { + + if all { + switch v := interface{}(m.GetExternalDataSource()).(type) { + case interface{ ValidateAll() error }: + if err := v.ValidateAll(); err != nil { + errors = append(errors, FileValidationError{ + field: "ExternalDataSource", + reason: "embedded message failed validation", + cause: err, + }) + } + case interface{ Validate() error }: + if err := v.Validate(); err != nil { + errors = append(errors, FileValidationError{ + field: "ExternalDataSource", + reason: "embedded message failed validation", + cause: err, + }) + } + } + } else if v, ok := interface{}(m.GetExternalDataSource()).(interface{ Validate() error }); ok { + if err := v.Validate(); err != nil { + return FileValidationError{ + field: "ExternalDataSource", + reason: "embedded message failed validation", + cause: err, + } + } + } + + } + if len(errors) > 0 { return FileMultiError(errors) } @@ -1471,6 +1504,110 @@ var _ interface { ErrorName() string } = FileValidationError{} +// Validate checks the field values on ExternalDataSource with the rules +// defined in the proto definition for this message. If any rules are +// violated, the first error encountered is returned, or nil if there are no violations. +func (m *ExternalDataSource) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on ExternalDataSource with the rules +// defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// ExternalDataSourceMultiError, or nil if none found. +func (m *ExternalDataSource) ValidateAll() error { + return m.validate(true) +} + +func (m *ExternalDataSource) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + // no validation rules for Location + + if len(errors) > 0 { + return ExternalDataSourceMultiError(errors) + } + + return nil +} + +// ExternalDataSourceMultiError is an error wrapping multiple validation errors +// returned by ExternalDataSource.ValidateAll() if the designated constraints +// aren't met. +type ExternalDataSourceMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m ExternalDataSourceMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m ExternalDataSourceMultiError) AllErrors() []error { return m } + +// ExternalDataSourceValidationError is the validation error returned by +// ExternalDataSource.Validate if the designated constraints aren't met. +type ExternalDataSourceValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e ExternalDataSourceValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e ExternalDataSourceValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e ExternalDataSourceValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e ExternalDataSourceValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e ExternalDataSourceValidationError) ErrorName() string { + return "ExternalDataSourceValidationError" +} + +// Error satisfies the builtin error interface +func (e ExternalDataSourceValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sExternalDataSource.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = ExternalDataSourceValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = ExternalDataSourceValidationError{} + // Validate checks the field values on GetFileRequest with the rules defined in // the proto definition for this message. If any rules are violated, the first // error encountered is returned, or nil if there are no violations. diff --git a/api/grpc/mpi/v1/files.proto b/api/grpc/mpi/v1/files.proto index 8af9a60bf..c6e0ebe56 100644 --- a/api/grpc/mpi/v1/files.proto +++ b/api/grpc/mpi/v1/files.proto @@ -130,6 +130,13 @@ message File { FileMeta file_meta = 1; // Unmanaged files will not be modified bool unmanaged = 2; + // external file source + optional ExternalDataSource external_data_source = 3; +} + +message ExternalDataSource { + // URL to the location of an external file + string location = 1; } // Represents the get file request diff --git a/docs/proto/protos.md b/docs/proto/protos.md index 874107b5b..3ae57f5e4 100644 --- a/docs/proto/protos.md +++ b/docs/proto/protos.md @@ -18,6 +18,7 @@ - [CertificateDates](#mpi-v1-CertificateDates) - [CertificateMeta](#mpi-v1-CertificateMeta) - [ConfigVersion](#mpi-v1-ConfigVersion) + - [ExternalDataSource](#mpi-v1-ExternalDataSource) - [File](#mpi-v1-File) - [FileContents](#mpi-v1-FileContents) - [FileDataChunk](#mpi-v1-FileDataChunk) @@ -295,6 +296,21 @@ Represents a specific configuration version associated with an instance + + +### ExternalDataSource + + + +| Field | Type | Label | Description | +| ----- | ---- | ----- | ----------- | +| location | [string](#string) | | URL to the location of an external file | + + + + + + ### File @@ -305,6 +321,7 @@ Represents meta data about a file | ----- | ---- | ----- | ----------- | | file_meta | [FileMeta](#mpi-v1-FileMeta) | | Meta information about the file, the name (including path) and hash | | unmanaged | [bool](#bool) | | Unmanaged files will not be modified | +| external_data_source | [ExternalDataSource](#mpi-v1-ExternalDataSource) | optional | external file source | diff --git a/test/mock/grpc/README.md b/test/mock/grpc/README.md index cb10c1c92..dae3cdcd7 100644 --- a/test/mock/grpc/README.md +++ b/test/mock/grpc/README.md @@ -413,7 +413,6 @@ Example request body to get stream upstreams example: } } ``` - ## POST /api/v1/instance/\/config/apply Used to send management plane config apply request over the Subscribe rpc stream to the NGINX Agent for a particular data plane instance. @@ -425,3 +424,73 @@ So the full path to the file used by the mock management plane would be `/tmp/co Simply edit this file and then perform a POST request against the `/api/v1/instance//config/apply` endpoint to execute a config apply request. +We can also send a JSON body with this POST request to point to a configuration file located on an external server. + +``` +POST /api/v1/instance/\/config/apply -H "Content-Type:application/json" -d +{ + "externalDataSources": [ + { + "filePath": "/path/to/my/file.txt", + "location": "s3://my-bucket/data/" + }, + { + "filePath": "/etc/nginx/secret.pem", + "location": "http://localhost:9092/external/secret.pem" + } + ] +} +``` + +## GET /api/v1/externalfile/\ + +Used to get files from a directory which can be used to mock an external file server. + +For example if we set the external file server as the location where our nginx-agent.conf file is located. + +Updated the Makefile: + +MOCK_MANAGEMENT_PLANE_EXTERNAL_FILE_SERVER ?= /Users//go/src/github.com/nginx/agent/ + +Sample API request: +GET /api/v1/externalfile/nginx-agent.conf + +Response: +``` +# +# /etc/nginx-agent/nginx-agent.conf +# +# Configuration file for NGINX Agent. +# + +log: + # set log level (error, warn, info, debug; default "info") + level: info + # set log path. if empty, don't log to file. + path: /var/log/nginx-agent/ + +allowed_directories: + - /etc/nginx + - /etc/app_protect + - /usr/local/etc/nginx + - /usr/share/nginx/modules + - /var/run/nginx + - /var/log/nginx +# +# Command server settings to connect to a management plane server +# +#command: +# server: +# host: "agent.connect.nginx.com" +# port: 443 +# auth: +# token: "" +# tls: +# skip_verify: false +command: + server: + host: localhost + port: 9091 + +``` + diff --git a/test/mock/grpc/mock_management_command_service.go b/test/mock/grpc/mock_management_command_service.go index 5f2115405..f3dbc4d0f 100644 --- a/test/mock/grpc/mock_management_command_service.go +++ b/test/mock/grpc/mock_management_command_service.go @@ -9,6 +9,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "io" "log/slog" "net" @@ -54,7 +55,8 @@ func init() { func NewCommandService( requestChan chan *mpi.ManagementPlaneRequest, configDirectory string, - externalFileServer string) *CommandService { + externalFileServer string, +) *CommandService { cs := &CommandService{ requestChan: requestChan, connectionMutex: sync.Mutex{}, @@ -80,6 +82,17 @@ func NewCommandService( return cs } +// Adding a struct to represent the external data source. +type ExternalDataSource struct { + FilePath string `json:"filePath"` + Location string `json:"location"` +} + +// Adding a struct for the request body of the config apply endpoint. +type ConfigApplyRequestBody struct { + ExternalDataSources []*ExternalDataSource `json:"externalDataSources"` +} + func (cs *CommandService) StartServer(listener net.Listener) { slog.Info("Starting mock management plane http server", "address", listener.Addr().String()) err := cs.server.RunListener(listener) @@ -365,7 +378,15 @@ func (cs *CommandService) addConfigApplyEndpoint() { return } - cs.instanceFiles[instanceID] = configFiles + updatedConfigFiles, externalFilesWereUpdated, err := processConfigApplyRequestBody(c, configFiles) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + + if externalFilesWereUpdated { + cs.instanceFiles[instanceID] = updatedConfigFiles + } request := mpi.ManagementPlaneRequest{ MessageMeta: &mpi.MessageMeta{ @@ -495,3 +516,33 @@ func createFile(fullPath, filePath string) (*mpi.File, error) { func isValidFile(info os.FileInfo, fileFullPath string) bool { return !info.IsDir() && !strings.HasSuffix(fileFullPath, ".DS_Store") } + +func processConfigApplyRequestBody(c *gin.Context, initialFiles []*mpi.File) ([]*mpi.File, bool, error) { + if c.Request.ContentLength == 0 { + return initialFiles, false, nil + } + + var body ConfigApplyRequestBody + if bindErr := c.BindJSON(&body); bindErr != nil { + return initialFiles, false, fmt.Errorf("invalid request body: %w", bindErr) + } + + filesMap := make(map[string]*mpi.File) + for _, file := range initialFiles { + if file.GetFileMeta() != nil { + filesMap[file.GetFileMeta().GetName()] = file + } + } + + var externalFilesWereUpdated bool + for _, ed := range body.ExternalDataSources { + if file, ok := filesMap[ed.FilePath]; ok { + file.ExternalDataSource = &mpi.ExternalDataSource{ + Location: ed.Location, + } + externalFilesWereUpdated = true + } + } + + return initialFiles, externalFilesWereUpdated, nil +} diff --git a/test/mock/grpc/mock_management_server.go b/test/mock/grpc/mock_management_server.go index 233c2b18e..93bb5ae9f 100644 --- a/test/mock/grpc/mock_management_server.go +++ b/test/mock/grpc/mock_management_server.go @@ -18,7 +18,7 @@ import ( "syscall" "time" - v1 "github.com/nginx/agent/v3/api/grpc/mpi/v1" + "github.com/nginx/agent/v3/api/grpc/mpi/v1" "github.com/nginx/agent/v3/internal/config" "github.com/bufbuild/protovalidate-go" From a3c1df4da46b9d125a25ac329284a7a259e6d39d Mon Sep 17 00:00:00 2001 From: Akshay Chawla <50681751+Akshay2191@users.noreply.github.com> Date: Mon, 1 Sep 2025 11:14:56 +0100 Subject: [PATCH 3/6] Fix traversal attack Adding validation to the user defined file path for external file server to stop possible traversal attacks. Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- .../grpc/mock_management_command_service.go | 24 ++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/test/mock/grpc/mock_management_command_service.go b/test/mock/grpc/mock_management_command_service.go index f3dbc4d0f..f64734a9f 100644 --- a/test/mock/grpc/mock_management_command_service.go +++ b/test/mock/grpc/mock_management_command_service.go @@ -441,17 +441,35 @@ func (cs *CommandService) addExternalFileServerEndpoint() { // This API will serve individual files from the external directory cs.server.GET("/api/v1/externalfile/:filename", func(c *gin.Context) { filename := c.Param("filename") + // Validate that the filename does not contain path separators or ".." + if strings.Contains(filename, "/") || strings.Contains(filename, "\\") || strings.Contains(filename, "..") { + c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid file name"}) + return + } filePath := filepath.Join(cs.externalFileServer, filename) + absBase, err := filepath.Abs(cs.externalFileServer) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Internal error"}) + return + } + absFile, err := filepath.Abs(filePath) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid file name"}) + return + } + if !strings.HasPrefix(absFile, absBase) { + c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid file name"}) + return + } // Check if the file exists - if _, err := os.Stat(filePath); os.IsNotExist(err) { + if _, err := os.Stat(absFile); os.IsNotExist(err) { c.JSON(http.StatusNotFound, gin.H{"error": "File not found"}) - return } // Serve the file - c.File(filePath) + c.File(absFile) }) slog.Info("Serving individual external files from", "directory", cs.externalFileServer) From d49a8514a56b458288d067f86b410f465070ed62 Mon Sep 17 00:00:00 2001 From: Akshay Chawla Date: Mon, 1 Sep 2025 11:56:01 +0100 Subject: [PATCH 4/6] fixing lint error --- test/mock/grpc/mock_management_command_service.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/mock/grpc/mock_management_command_service.go b/test/mock/grpc/mock_management_command_service.go index f64734a9f..ac527033d 100644 --- a/test/mock/grpc/mock_management_command_service.go +++ b/test/mock/grpc/mock_management_command_service.go @@ -463,7 +463,7 @@ func (cs *CommandService) addExternalFileServerEndpoint() { } // Check if the file exists - if _, err := os.Stat(absFile); os.IsNotExist(err) { + if _, file_err := os.Stat(absFile); os.IsNotExist(file_err) { c.JSON(http.StatusNotFound, gin.H{"error": "File not found"}) return } From 1803f90cf687fe52a4a79c0af0ebe5147535bf7e Mon Sep 17 00:00:00 2001 From: Akshay Chawla Date: Tue, 9 Sep 2025 11:45:03 +0100 Subject: [PATCH 5/6] fixing integration test --- test/mock/grpc/mock_management_command_service.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/mock/grpc/mock_management_command_service.go b/test/mock/grpc/mock_management_command_service.go index ac527033d..98211b705 100644 --- a/test/mock/grpc/mock_management_command_service.go +++ b/test/mock/grpc/mock_management_command_service.go @@ -386,6 +386,8 @@ func (cs *CommandService) addConfigApplyEndpoint() { if externalFilesWereUpdated { cs.instanceFiles[instanceID] = updatedConfigFiles + } else { + cs.instanceFiles[instanceID] = configFiles } request := mpi.ManagementPlaneRequest{ From a97988cf58af6591b8690d2d82afaa9d2d51baea Mon Sep 17 00:00:00 2001 From: Akshay Chawla Date: Wed, 10 Sep 2025 10:08:02 +0100 Subject: [PATCH 6/6] worked on PR review comments --- api/grpc/mpi/v1/command.pb.go | 2 +- api/grpc/mpi/v1/common.pb.go | 2 +- api/grpc/mpi/v1/files.pb.go | 2 +- test/mock/grpc/cmd/main.go | 3 +- .../grpc/mock_management_command_service.go | 80 ++++++++++++------- 5 files changed, 56 insertions(+), 33 deletions(-) diff --git a/api/grpc/mpi/v1/command.pb.go b/api/grpc/mpi/v1/command.pb.go index 39ef93489..b5f0346f4 100644 --- a/api/grpc/mpi/v1/command.pb.go +++ b/api/grpc/mpi/v1/command.pb.go @@ -8,7 +8,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.8 +// protoc-gen-go v1.36.9 // protoc (unknown) // source: mpi/v1/command.proto diff --git a/api/grpc/mpi/v1/common.pb.go b/api/grpc/mpi/v1/common.pb.go index 890c2b073..e6d06cf37 100644 --- a/api/grpc/mpi/v1/common.pb.go +++ b/api/grpc/mpi/v1/common.pb.go @@ -5,7 +5,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.8 +// protoc-gen-go v1.36.9 // protoc (unknown) // source: mpi/v1/common.proto diff --git a/api/grpc/mpi/v1/files.pb.go b/api/grpc/mpi/v1/files.pb.go index 4fe57bb7c..47d1362b7 100644 --- a/api/grpc/mpi/v1/files.pb.go +++ b/api/grpc/mpi/v1/files.pb.go @@ -5,7 +5,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.8 +// protoc-gen-go v1.36.9 // protoc (unknown) // source: mpi/v1/files.proto diff --git a/test/mock/grpc/cmd/main.go b/test/mock/grpc/cmd/main.go index a50f034ac..06f6ff732 100644 --- a/test/mock/grpc/cmd/main.go +++ b/test/mock/grpc/cmd/main.go @@ -79,7 +79,7 @@ func main() { defaultExternalFileServer, externalFileServerErr := generateDefaultExternalFileSevrevDirectory() externalFileServer = &defaultExternalFileServer if externalFileServerErr != nil { - slog.ErrorContext(ctx, "Failed to create default config directory", "error", err) + slog.ErrorContext(ctx, "Failed to create external file server directory", "error", err) os.Exit(1) } } @@ -117,7 +117,6 @@ func generateDefaultExternalFileSevrevDirectory() (string, error) { err := os.MkdirAll(externalFileServer, directoryPermissions) if err != nil { - slog.Error("Failed to create external file server directory", "error", err) return "", err } diff --git a/test/mock/grpc/mock_management_command_service.go b/test/mock/grpc/mock_management_command_service.go index 98211b705..f68c4c7cd 100644 --- a/test/mock/grpc/mock_management_command_service.go +++ b/test/mock/grpc/mock_management_command_service.go @@ -378,13 +378,13 @@ func (cs *CommandService) addConfigApplyEndpoint() { return } - updatedConfigFiles, externalFilesWereUpdated, err := processConfigApplyRequestBody(c, configFiles) + updatedConfigFiles, externalFilesUpdated, err := processConfigApplyRequestBody(c, configFiles) if err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) return } - if externalFilesWereUpdated { + if externalFilesUpdated { cs.instanceFiles[instanceID] = updatedConfigFiles } else { cs.instanceFiles[instanceID] = configFiles @@ -443,38 +443,22 @@ func (cs *CommandService) addExternalFileServerEndpoint() { // This API will serve individual files from the external directory cs.server.GET("/api/v1/externalfile/:filename", func(c *gin.Context) { filename := c.Param("filename") - // Validate that the filename does not contain path separators or ".." - if strings.Contains(filename, "/") || strings.Contains(filename, "\\") || strings.Contains(filename, "..") { - c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid file name"}) - return - } - filePath := filepath.Join(cs.externalFileServer, filename) - absBase, err := filepath.Abs(cs.externalFileServer) - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": "Internal error"}) - return - } - absFile, err := filepath.Abs(filePath) + + absFile, err := validateFile(cs.externalFileServer, filename) if err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid file name"}) - return - } - if !strings.HasPrefix(absFile, absBase) { - c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid file name"}) - return - } + if errors.Is(err, os.ErrNotExist) { + c.JSON(http.StatusNotFound, gin.H{"error": "File not found"}) + } else { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + } - // Check if the file exists - if _, file_err := os.Stat(absFile); os.IsNotExist(file_err) { - c.JSON(http.StatusNotFound, gin.H{"error": "File not found"}) return } - // Serve the file c.File(absFile) }) - slog.Info("Serving individual external files from", "directory", cs.externalFileServer) + slog.Info("Serving individual external files from", "external_file_server", cs.externalFileServer) } func (cs *CommandService) findInstanceConfigFiles(instanceID string) (configFiles []*mpi.File, err error) { @@ -537,6 +521,34 @@ func isValidFile(info os.FileInfo, fileFullPath string) bool { return !info.IsDir() && !strings.HasSuffix(fileFullPath, ".DS_Store") } +func validateFile(externalFileServer, filename string) (string, error) { + if strings.Contains(filename, "/") || strings.Contains(filename, "\\") || strings.Contains(filename, "..") { + return "", errors.New("invalid file name") + } + + filePath := filepath.Join(externalFileServer, filename) + + absBase, err := filepath.Abs(externalFileServer) + if err != nil { + return "", fmt.Errorf("internal error: %w", err) + } + + absFile, err := filepath.Abs(filePath) + if err != nil { + return "", fmt.Errorf("invalid file name: %w", err) + } + + if !strings.HasPrefix(absFile, absBase) { + return "", errors.New("invalid file name") + } + + if _, fileErr := os.Stat(absFile); os.IsNotExist(fileErr) { + return "", os.ErrNotExist + } + + return absFile, nil +} + func processConfigApplyRequestBody(c *gin.Context, initialFiles []*mpi.File) ([]*mpi.File, bool, error) { if c.Request.ContentLength == 0 { return initialFiles, false, nil @@ -555,14 +567,26 @@ func processConfigApplyRequestBody(c *gin.Context, initialFiles []*mpi.File) ([] } var externalFilesWereUpdated bool + updatedFiles := initialFiles + for _, ed := range body.ExternalDataSources { if file, ok := filesMap[ed.FilePath]; ok { file.ExternalDataSource = &mpi.ExternalDataSource{ Location: ed.Location, } - externalFilesWereUpdated = true + } else { + newFile := &mpi.File{ + FileMeta: &mpi.FileMeta{ + Name: ed.FilePath, + }, + ExternalDataSource: &mpi.ExternalDataSource{ + Location: ed.Location, + }, + } + updatedFiles = append(updatedFiles, newFile) } + externalFilesWereUpdated = true } - return initialFiles, externalFilesWereUpdated, nil + return updatedFiles, externalFilesWereUpdated, nil }