diff --git a/changes/28788-name-software-checksum b/changes/28788-name-software-checksum new file mode 100644 index 000000000000..b8624a6b1a34 --- /dev/null +++ b/changes/28788-name-software-checksum @@ -0,0 +1 @@ +* Added software name into checksum calculation for macos apps \ No newline at end of file diff --git a/server/datastore/mysql/migrations/tables/20251010153829_AddNameToSoftwareCheckumCalculation.go b/server/datastore/mysql/migrations/tables/20251010153829_AddNameToSoftwareCheckumCalculation.go new file mode 100644 index 000000000000..f9a6a4cc6518 --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20251010153829_AddNameToSoftwareCheckumCalculation.go @@ -0,0 +1,70 @@ +package tables + +import ( + "database/sql" + "fmt" +) + +func init() { + MigrationClient.AddMigration(Up_20251010153829, Down_20251010153829) +} + +func Up_20251010153829(tx *sql.Tx) error { + var minID, maxID sql.NullInt64 + err := tx.QueryRow(` + SELECT MIN(id), MAX(id) + FROM software + WHERE source = 'apps' + AND bundle_identifier IS NOT NULL + AND bundle_identifier != '' + `).Scan(&minID, &maxID) + if err != nil { + return fmt.Errorf("getting ID range: %w", err) + } + + if !minID.Valid || !maxID.Valid { + return nil + } + + const batchSize = 10000 + for startID := minID.Int64; startID <= maxID.Int64; startID += batchSize { + endID := startID + batchSize - 1 + if endID > maxID.Int64 { + endID = maxID.Int64 + } + + softwareStmt := ` + UPDATE software SET + checksum = UNHEX( + MD5( + -- concatenate with separator \x00 + CONCAT_WS(CHAR(0), + version, + source, + bundle_identifier, + ` + "`release`" + `, + arch, + vendor, + extension_for, + extension_id, + name + ) + ) + ) + WHERE source = 'apps' + AND bundle_identifier IS NOT NULL + AND bundle_identifier != '' + AND id >= ? AND id <= ? + ` + _, err = tx.Exec(softwareStmt, startID, endID) + if err != nil { + return fmt.Errorf("updating software checksums (batch %d-%d): %w", startID, endID, err) + } + } + + return nil +} + +func Down_20251010153829(tx *sql.Tx) error { + return nil +} diff --git a/server/datastore/mysql/migrations/tables/20251010153829_AddNameToSoftwareCheckumCalculation_test.go b/server/datastore/mysql/migrations/tables/20251010153829_AddNameToSoftwareCheckumCalculation_test.go new file mode 100644 index 000000000000..d8c94c0891a0 --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20251010153829_AddNameToSoftwareCheckumCalculation_test.go @@ -0,0 +1,122 @@ +package tables + +import ( + "crypto/md5" //nolint:gosec // MD5 is used for checksums, not security + "database/sql" + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestUp_20251010153829(t *testing.T) { + db := applyUpToPrev(t) + + computeOldChecksum := func(name, version, source, bundleID, release, arch, vendor, extensionFor, extensionID string) []byte { + h := md5.New() //nolint:gosec + cols := []string{version, source, bundleID, release, arch, vendor, extensionFor, extensionID} + if source != "apps" { + cols = append([]string{name}, cols...) + } + _, _ = fmt.Fprint(h, strings.Join(cols, "\x00")) + return h.Sum(nil) + } + + computeNewChecksum := func(name, version, source, bundleID, release, arch, vendor, extensionFor, extensionID string) []byte { + h := md5.New() //nolint:gosec + cols := []string{version, source, bundleID, release, arch, vendor, extensionFor, extensionID, name} + _, _ = fmt.Fprint(h, strings.Join(cols, "\x00")) + return h.Sum(nil) + } + + insertTitle := `INSERT INTO software_titles (name, source, extension_for, bundle_identifier) VALUES (?, ?, ?, ?)` + result, err := db.Exec(insertTitle, "Test App", "apps", "", "com.test.app") + require.NoError(t, err) + titleID, err := result.LastInsertId() + require.NoError(t, err) + + insertSoftware := `INSERT INTO software + (name, version, source, bundle_identifier, ` + "`release`" + `, arch, vendor, extension_for, extension_id, checksum, title_id) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + + // software with bundle_identifier should be updated + app1Name := "GoLand.app" + app1BundleID := "com.jetbrains.goland" + app1OldChecksum := computeOldChecksum(app1Name, "2023.1", "apps", app1BundleID, "", "x86_64", "JetBrains", "", "") + app1NewChecksum := computeNewChecksum(app1Name, "2023.1", "apps", app1BundleID, "", "x86_64", "JetBrains", "", "") + _, err = db.Exec(insertSoftware, app1Name, "2023.1", "apps", app1BundleID, "", "x86_64", "JetBrains", "", "", app1OldChecksum, titleID) + require.NoError(t, err) + + app2Name := "GoLand 2.app" + app2BundleID := "com.jetbrains.goland" + app2OldChecksum := computeOldChecksum(app2Name, "2023.2", "apps", app2BundleID, "", "x86_64", "JetBrains", "", "") + app2NewChecksum := computeNewChecksum(app2Name, "2023.2", "apps", app2BundleID, "", "x86_64", "JetBrains", "", "") + _, err = db.Exec(insertSoftware, app2Name, "2023.2", "apps", app2BundleID, "", "x86_64", "JetBrains", "", "", app2OldChecksum, titleID) + require.NoError(t, err) + + // softwares without bundle_identifier - no update + app3Name := "SomeApp.app" + app3OldChecksum := computeOldChecksum(app3Name, "1.0", "apps", "", "", "x86_64", "Vendor", "", "") + _, err = db.Exec(insertSoftware, app3Name, "1.0", "apps", nil, "", "x86_64", "Vendor", "", "", app3OldChecksum, titleID) + require.NoError(t, err) + + app4Name := "AnotherApp.app" + app4OldChecksum := computeOldChecksum(app4Name, "2.0", "apps", "", "", "arm64", "Another Vendor", "", "") + _, err = db.Exec(insertSoftware, app4Name, "2.0", "apps", "", "", "arm64", "Another Vendor", "", "", app4OldChecksum, titleID) + require.NoError(t, err) + + // Windows software - no update + winName := "Notepad++" + winOldChecksum := computeOldChecksum(winName, "8.5.0", "programs", "", "", "x86_64", "Don Ho", "", "") + _, err = db.Exec(insertSoftware, winName, "8.5.0", "programs", nil, "", "x86_64", "Don Ho", "", "", winOldChecksum, titleID) + require.NoError(t, err) + + // Linux software - no update + linuxName := "vim" + linuxOldChecksum := computeOldChecksum(linuxName, "8.2", "deb_packages", "", "1ubuntu1", "amd64", "Ubuntu", "", "") + _, err = db.Exec(insertSoftware, linuxName, "8.2", "deb_packages", nil, "1ubuntu1", "amd64", "Ubuntu", "", "", linuxOldChecksum, titleID) + require.NoError(t, err) + + applyNext(t, db) + + type softwareRow struct { + Name string `db:"name"` + Source string `db:"source"` + BundleIdentifier sql.NullString `db:"bundle_identifier"` + Checksum []byte `db:"checksum"` + } + + var software []softwareRow + err = db.Select(&software, `SELECT name, source, bundle_identifier, checksum FROM software ORDER BY name`) + require.NoError(t, err) + require.Len(t, software, 6) + + for _, sw := range software { + switch sw.Name { + case app1Name: + require.Equal(t, app1NewChecksum, sw.Checksum) + require.True(t, sw.BundleIdentifier.Valid) + require.Equal(t, app1BundleID, sw.BundleIdentifier.String) + case app2Name: + require.Equal(t, app2NewChecksum, sw.Checksum) + require.True(t, sw.BundleIdentifier.Valid) + require.Equal(t, app2BundleID, sw.BundleIdentifier.String) + case app3Name: + require.Equal(t, app3OldChecksum, sw.Checksum) + require.False(t, sw.BundleIdentifier.Valid) + case app4Name: + require.Equal(t, app4OldChecksum, sw.Checksum) + require.True(t, sw.BundleIdentifier.Valid) + require.Equal(t, "", sw.BundleIdentifier.String) + case winName: + require.Equal(t, winOldChecksum, sw.Checksum) + require.Equal(t, "programs", sw.Source) + case linuxName: + require.Equal(t, linuxOldChecksum, sw.Checksum) + require.Equal(t, "deb_packages", sw.Source) + default: + t.Fatalf("Unexpected software entry: %s", sw.Name) + } + } +} diff --git a/server/datastore/mysql/schema.sql b/server/datastore/mysql/schema.sql index 2e8dd0568925..619bdbd5431e 100644 --- a/server/datastore/mysql/schema.sql +++ b/server/datastore/mysql/schema.sql @@ -1560,9 +1560,9 @@ CREATE TABLE `migration_status_tables` ( `is_applied` tinyint(1) NOT NULL, `tstamp` timestamp NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`) -) /*!50100 TABLESPACE `innodb_system` */ ENGINE=InnoDB AUTO_INCREMENT=428 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) /*!50100 TABLESPACE `innodb_system` */ ENGINE=InnoDB AUTO_INCREMENT=429 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; /*!40101 SET character_set_client = @saved_cs_client */; -INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'),(120,20220201084510,1,'2020-01-01 01:01:01'),(121,20220208144830,1,'2020-01-01 01:01:01'),(122,20220208144831,1,'2020-01-01 01:01:01'),(123,20220215152203,1,'2020-01-01 01:01:01'),(124,20220223113157,1,'2020-01-01 01:01:01'),(125,20220307104655,1,'2020-01-01 01:01:01'),(126,20220309133956,1,'2020-01-01 01:01:01'),(127,20220316155700,1,'2020-01-01 01:01:01'),(128,20220323152301,1,'2020-01-01 01:01:01'),(129,20220330100659,1,'2020-01-01 01:01:01'),(130,20220404091216,1,'2020-01-01 01:01:01'),(131,20220419140750,1,'2020-01-01 01:01:01'),(132,20220428140039,1,'2020-01-01 01:01:01'),(133,20220503134048,1,'2020-01-01 01:01:01'),(134,20220524102918,1,'2020-01-01 01:01:01'),(135,20220526123327,1,'2020-01-01 01:01:01'),(136,20220526123328,1,'2020-01-01 01:01:01'),(137,20220526123329,1,'2020-01-01 01:01:01'),(138,20220608113128,1,'2020-01-01 01:01:01'),(139,20220627104817,1,'2020-01-01 01:01:01'),(140,20220704101843,1,'2020-01-01 01:01:01'),(141,20220708095046,1,'2020-01-01 01:01:01'),(142,20220713091130,1,'2020-01-01 01:01:01'),(143,20220802135510,1,'2020-01-01 01:01:01'),(144,20220818101352,1,'2020-01-01 01:01:01'),(145,20220822161445,1,'2020-01-01 01:01:01'),(146,20220831100036,1,'2020-01-01 01:01:01'),(147,20220831100151,1,'2020-01-01 01:01:01'),(148,20220908181826,1,'2020-01-01 01:01:01'),(149,20220914154915,1,'2020-01-01 01:01:01'),(150,20220915165115,1,'2020-01-01 01:01:01'),(151,20220915165116,1,'2020-01-01 01:01:01'),(152,20220928100158,1,'2020-01-01 01:01:01'),(153,20221014084130,1,'2020-01-01 01:01:01'),(154,20221027085019,1,'2020-01-01 01:01:01'),(155,20221101103952,1,'2020-01-01 01:01:01'),(156,20221104144401,1,'2020-01-01 01:01:01'),(157,20221109100749,1,'2020-01-01 01:01:01'),(158,20221115104546,1,'2020-01-01 01:01:01'),(159,20221130114928,1,'2020-01-01 01:01:01'),(160,20221205112142,1,'2020-01-01 01:01:01'),(161,20221216115820,1,'2020-01-01 01:01:01'),(162,20221220195934,1,'2020-01-01 01:01:01'),(163,20221220195935,1,'2020-01-01 01:01:01'),(164,20221223174807,1,'2020-01-01 01:01:01'),(165,20221227163855,1,'2020-01-01 01:01:01'),(166,20221227163856,1,'2020-01-01 01:01:01'),(167,20230202224725,1,'2020-01-01 01:01:01'),(168,20230206163608,1,'2020-01-01 01:01:01'),(169,20230214131519,1,'2020-01-01 01:01:01'),(170,20230303135738,1,'2020-01-01 01:01:01'),(171,20230313135301,1,'2020-01-01 01:01:01'),(172,20230313141819,1,'2020-01-01 01:01:01'),(173,20230315104937,1,'2020-01-01 01:01:01'),(174,20230317173844,1,'2020-01-01 01:01:01'),(175,20230320133602,1,'2020-01-01 01:01:01'),(176,20230330100011,1,'2020-01-01 01:01:01'),(177,20230330134823,1,'2020-01-01 01:01:01'),(178,20230405232025,1,'2020-01-01 01:01:01'),(179,20230408084104,1,'2020-01-01 01:01:01'),(180,20230411102858,1,'2020-01-01 01:01:01'),(181,20230421155932,1,'2020-01-01 01:01:01'),(182,20230425082126,1,'2020-01-01 01:01:01'),(183,20230425105727,1,'2020-01-01 01:01:01'),(184,20230501154913,1,'2020-01-01 01:01:01'),(185,20230503101418,1,'2020-01-01 01:01:01'),(186,20230515144206,1,'2020-01-01 01:01:01'),(187,20230517140952,1,'2020-01-01 01:01:01'),(188,20230517152807,1,'2020-01-01 01:01:01'),(189,20230518114155,1,'2020-01-01 01:01:01'),(190,20230520153236,1,'2020-01-01 01:01:01'),(191,20230525151159,1,'2020-01-01 01:01:01'),(192,20230530122103,1,'2020-01-01 01:01:01'),(193,20230602111827,1,'2020-01-01 01:01:01'),(194,20230608103123,1,'2020-01-01 01:01:01'),(195,20230629140529,1,'2020-01-01 01:01:01'),(196,20230629140530,1,'2020-01-01 01:01:01'),(197,20230711144622,1,'2020-01-01 01:01:01'),(198,20230721135421,1,'2020-01-01 01:01:01'),(199,20230721161508,1,'2020-01-01 01:01:01'),(200,20230726115701,1,'2020-01-01 01:01:01'),(201,20230807100822,1,'2020-01-01 01:01:01'),(202,20230814150442,1,'2020-01-01 01:01:01'),(203,20230823122728,1,'2020-01-01 01:01:01'),(204,20230906152143,1,'2020-01-01 01:01:01'),(205,20230911163618,1,'2020-01-01 01:01:01'),(206,20230912101759,1,'2020-01-01 01:01:01'),(207,20230915101341,1,'2020-01-01 01:01:01'),(208,20230918132351,1,'2020-01-01 01:01:01'),(209,20231004144339,1,'2020-01-01 01:01:01'),(210,20231009094541,1,'2020-01-01 01:01:01'),(211,20231009094542,1,'2020-01-01 01:01:01'),(212,20231009094543,1,'2020-01-01 01:01:01'),(213,20231009094544,1,'2020-01-01 01:01:01'),(214,20231016091915,1,'2020-01-01 01:01:01'),(215,20231024174135,1,'2020-01-01 01:01:01'),(216,20231025120016,1,'2020-01-01 01:01:01'),(217,20231025160156,1,'2020-01-01 01:01:01'),(218,20231031165350,1,'2020-01-01 01:01:01'),(219,20231106144110,1,'2020-01-01 01:01:01'),(220,20231107130934,1,'2020-01-01 01:01:01'),(221,20231109115838,1,'2020-01-01 01:01:01'),(222,20231121054530,1,'2020-01-01 01:01:01'),(223,20231122101320,1,'2020-01-01 01:01:01'),(224,20231130132828,1,'2020-01-01 01:01:01'),(225,20231130132931,1,'2020-01-01 01:01:01'),(226,20231204155427,1,'2020-01-01 01:01:01'),(227,20231206142340,1,'2020-01-01 01:01:01'),(228,20231207102320,1,'2020-01-01 01:01:01'),(229,20231207102321,1,'2020-01-01 01:01:01'),(230,20231207133731,1,'2020-01-01 01:01:01'),(231,20231212094238,1,'2020-01-01 01:01:01'),(232,20231212095734,1,'2020-01-01 01:01:01'),(233,20231212161121,1,'2020-01-01 01:01:01'),(234,20231215122713,1,'2020-01-01 01:01:01'),(235,20231219143041,1,'2020-01-01 01:01:01'),(236,20231224070653,1,'2020-01-01 01:01:01'),(237,20240110134315,1,'2020-01-01 01:01:01'),(238,20240119091637,1,'2020-01-01 01:01:01'),(239,20240126020642,1,'2020-01-01 01:01:01'),(240,20240126020643,1,'2020-01-01 01:01:01'),(241,20240129162819,1,'2020-01-01 01:01:01'),(242,20240130115133,1,'2020-01-01 01:01:01'),(243,20240131083822,1,'2020-01-01 01:01:01'),(244,20240205095928,1,'2020-01-01 01:01:01'),(245,20240205121956,1,'2020-01-01 01:01:01'),(246,20240209110212,1,'2020-01-01 01:01:01'),(247,20240212111533,1,'2020-01-01 01:01:01'),(248,20240221112844,1,'2020-01-01 01:01:01'),(249,20240222073518,1,'2020-01-01 01:01:01'),(250,20240222135115,1,'2020-01-01 01:01:01'),(251,20240226082255,1,'2020-01-01 01:01:01'),(252,20240228082706,1,'2020-01-01 01:01:01'),(253,20240301173035,1,'2020-01-01 01:01:01'),(254,20240302111134,1,'2020-01-01 01:01:01'),(255,20240312103753,1,'2020-01-01 01:01:01'),(256,20240313143416,1,'2020-01-01 01:01:01'),(257,20240314085226,1,'2020-01-01 01:01:01'),(258,20240314151747,1,'2020-01-01 01:01:01'),(259,20240320145650,1,'2020-01-01 01:01:01'),(260,20240327115530,1,'2020-01-01 01:01:01'),(261,20240327115617,1,'2020-01-01 01:01:01'),(262,20240408085837,1,'2020-01-01 01:01:01'),(263,20240415104633,1,'2020-01-01 01:01:01'),(264,20240430111727,1,'2020-01-01 01:01:01'),(265,20240515200020,1,'2020-01-01 01:01:01'),(266,20240521143023,1,'2020-01-01 01:01:01'),(267,20240521143024,1,'2020-01-01 01:01:01'),(268,20240601174138,1,'2020-01-01 01:01:01'),(269,20240607133721,1,'2020-01-01 01:01:01'),(270,20240612150059,1,'2020-01-01 01:01:01'),(271,20240613162201,1,'2020-01-01 01:01:01'),(272,20240613172616,1,'2020-01-01 01:01:01'),(273,20240618142419,1,'2020-01-01 01:01:01'),(274,20240625093543,1,'2020-01-01 01:01:01'),(275,20240626195531,1,'2020-01-01 01:01:01'),(276,20240702123921,1,'2020-01-01 01:01:01'),(277,20240703154849,1,'2020-01-01 01:01:01'),(278,20240707134035,1,'2020-01-01 01:01:01'),(279,20240707134036,1,'2020-01-01 01:01:01'),(280,20240709124958,1,'2020-01-01 01:01:01'),(281,20240709132642,1,'2020-01-01 01:01:01'),(282,20240709183940,1,'2020-01-01 01:01:01'),(283,20240710155623,1,'2020-01-01 01:01:01'),(284,20240723102712,1,'2020-01-01 01:01:01'),(285,20240725152735,1,'2020-01-01 01:01:01'),(286,20240725182118,1,'2020-01-01 01:01:01'),(287,20240726100517,1,'2020-01-01 01:01:01'),(288,20240730171504,1,'2020-01-01 01:01:01'),(289,20240730174056,1,'2020-01-01 01:01:01'),(290,20240730215453,1,'2020-01-01 01:01:01'),(291,20240730374423,1,'2020-01-01 01:01:01'),(292,20240801115359,1,'2020-01-01 01:01:01'),(293,20240802101043,1,'2020-01-01 01:01:01'),(294,20240802113716,1,'2020-01-01 01:01:01'),(295,20240814135330,1,'2020-01-01 01:01:01'),(296,20240815000000,1,'2020-01-01 01:01:01'),(297,20240815000001,1,'2020-01-01 01:01:01'),(298,20240816103247,1,'2020-01-01 01:01:01'),(299,20240820091218,1,'2020-01-01 01:01:01'),(300,20240826111228,1,'2020-01-01 01:01:01'),(301,20240826160025,1,'2020-01-01 01:01:01'),(302,20240829165448,1,'2020-01-01 01:01:01'),(303,20240829165605,1,'2020-01-01 01:01:01'),(304,20240829165715,1,'2020-01-01 01:01:01'),(305,20240829165930,1,'2020-01-01 01:01:01'),(306,20240829170023,1,'2020-01-01 01:01:01'),(307,20240829170033,1,'2020-01-01 01:01:01'),(308,20240829170044,1,'2020-01-01 01:01:01'),(309,20240905105135,1,'2020-01-01 01:01:01'),(310,20240905140514,1,'2020-01-01 01:01:01'),(311,20240905200000,1,'2020-01-01 01:01:01'),(312,20240905200001,1,'2020-01-01 01:01:01'),(313,20241002104104,1,'2020-01-01 01:01:01'),(314,20241002104105,1,'2020-01-01 01:01:01'),(315,20241002104106,1,'2020-01-01 01:01:01'),(316,20241002210000,1,'2020-01-01 01:01:01'),(317,20241003145349,1,'2020-01-01 01:01:01'),(318,20241004005000,1,'2020-01-01 01:01:01'),(319,20241008083925,1,'2020-01-01 01:01:01'),(320,20241009090010,1,'2020-01-01 01:01:01'),(321,20241017163402,1,'2020-01-01 01:01:01'),(322,20241021224359,1,'2020-01-01 01:01:01'),(323,20241022140321,1,'2020-01-01 01:01:01'),(324,20241025111236,1,'2020-01-01 01:01:01'),(325,20241025112748,1,'2020-01-01 01:01:01'),(326,20241025141855,1,'2020-01-01 01:01:01'),(327,20241110152839,1,'2020-01-01 01:01:01'),(328,20241110152840,1,'2020-01-01 01:01:01'),(329,20241110152841,1,'2020-01-01 01:01:01'),(330,20241116233322,1,'2020-01-01 01:01:01'),(331,20241122171434,1,'2020-01-01 01:01:01'),(332,20241125150614,1,'2020-01-01 01:01:01'),(333,20241203125346,1,'2020-01-01 01:01:01'),(334,20241203130032,1,'2020-01-01 01:01:01'),(335,20241205122800,1,'2020-01-01 01:01:01'),(336,20241209164540,1,'2020-01-01 01:01:01'),(337,20241210140021,1,'2020-01-01 01:01:01'),(338,20241219180042,1,'2020-01-01 01:01:01'),(339,20241220100000,1,'2020-01-01 01:01:01'),(340,20241220114903,1,'2020-01-01 01:01:01'),(341,20241220114904,1,'2020-01-01 01:01:01'),(342,20241224000000,1,'2020-01-01 01:01:01'),(343,20241230000000,1,'2020-01-01 01:01:01'),(344,20241231112624,1,'2020-01-01 01:01:01'),(345,20250102121439,1,'2020-01-01 01:01:01'),(346,20250121094045,1,'2020-01-01 01:01:01'),(347,20250121094500,1,'2020-01-01 01:01:01'),(348,20250121094600,1,'2020-01-01 01:01:01'),(349,20250121094700,1,'2020-01-01 01:01:01'),(350,20250124194347,1,'2020-01-01 01:01:01'),(351,20250127162751,1,'2020-01-01 01:01:01'),(352,20250213104005,1,'2020-01-01 01:01:01'),(353,20250214205657,1,'2020-01-01 01:01:01'),(354,20250217093329,1,'2020-01-01 01:01:01'),(355,20250219090511,1,'2020-01-01 01:01:01'),(356,20250219100000,1,'2020-01-01 01:01:01'),(357,20250219142401,1,'2020-01-01 01:01:01'),(358,20250224184002,1,'2020-01-01 01:01:01'),(359,20250225085436,1,'2020-01-01 01:01:01'),(360,20250226000000,1,'2020-01-01 01:01:01'),(361,20250226153445,1,'2020-01-01 01:01:01'),(362,20250304162702,1,'2020-01-01 01:01:01'),(363,20250306144233,1,'2020-01-01 01:01:01'),(364,20250313163430,1,'2020-01-01 01:01:01'),(365,20250317130944,1,'2020-01-01 01:01:01'),(366,20250318165922,1,'2020-01-01 01:01:01'),(367,20250320132525,1,'2020-01-01 01:01:01'),(368,20250320200000,1,'2020-01-01 01:01:01'),(369,20250326161930,1,'2020-01-01 01:01:01'),(370,20250326161931,1,'2020-01-01 01:01:01'),(371,20250331042354,1,'2020-01-01 01:01:01'),(372,20250331154206,1,'2020-01-01 01:01:01'),(373,20250401155831,1,'2020-01-01 01:01:01'),(374,20250408133233,1,'2020-01-01 01:01:01'),(375,20250410104321,1,'2020-01-01 01:01:01'),(376,20250421085116,1,'2020-01-01 01:01:01'),(377,20250422095806,1,'2020-01-01 01:01:01'),(378,20250424153059,1,'2020-01-01 01:01:01'),(379,20250430103833,1,'2020-01-01 01:01:01'),(380,20250430112622,1,'2020-01-01 01:01:01'),(381,20250501162727,1,'2020-01-01 01:01:01'),(382,20250502154517,1,'2020-01-01 01:01:01'),(383,20250502222222,1,'2020-01-01 01:01:01'),(384,20250507170845,1,'2020-01-01 01:01:01'),(385,20250513162912,1,'2020-01-01 01:01:01'),(386,20250519161614,1,'2020-01-01 01:01:01'),(387,20250519170000,1,'2020-01-01 01:01:01'),(388,20250520153848,1,'2020-01-01 01:01:01'),(389,20250528115932,1,'2020-01-01 01:01:01'),(390,20250529102706,1,'2020-01-01 01:01:01'),(391,20250603105558,1,'2020-01-01 01:01:01'),(392,20250609102714,1,'2020-01-01 01:01:01'),(393,20250609112613,1,'2020-01-01 01:01:01'),(394,20250613103810,1,'2020-01-01 01:01:01'),(395,20250616193950,1,'2020-01-01 01:01:01'),(396,20250624140757,1,'2020-01-01 01:01:01'),(397,20250626130239,1,'2020-01-01 01:01:01'),(398,20250629131032,1,'2020-01-01 01:01:01'),(399,20250701155654,1,'2020-01-01 01:01:01'),(400,20250707095725,1,'2020-01-01 01:01:01'),(401,20250716152435,1,'2020-01-01 01:01:01'),(402,20250718091828,1,'2020-01-01 01:01:01'),(403,20250728122229,1,'2020-01-01 01:01:01'),(404,20250731122715,1,'2020-01-01 01:01:01'),(405,20250731151000,1,'2020-01-01 01:01:01'),(406,20250803000000,1,'2020-01-01 01:01:01'),(407,20250805083116,1,'2020-01-01 01:01:01'),(408,20250807140441,1,'2020-01-01 01:01:01'),(409,20250808000000,1,'2020-01-01 01:01:01'),(410,20250811155036,1,'2020-01-01 01:01:01'),(411,20250813205039,1,'2020-01-01 01:01:01'),(412,20250814123333,1,'2020-01-01 01:01:01'),(413,20250815130115,1,'2020-01-01 01:01:01'),(414,20250816115553,1,'2020-01-01 01:01:01'),(415,20250817154557,1,'2020-01-01 01:01:01'),(416,20250825113751,1,'2020-01-01 01:01:01'),(417,20250827113140,1,'2020-01-01 01:01:01'),(418,20250828120836,1,'2020-01-01 01:01:01'),(419,20250902112642,1,'2020-01-01 01:01:01'),(420,20250904091745,1,'2020-01-01 01:01:01'),(421,20250905090000,1,'2020-01-01 01:01:01'),(422,20250922083056,1,'2020-01-01 01:01:01'),(423,20250923120000,1,'2020-01-01 01:01:01'),(424,20250926123048,1,'2020-01-01 01:01:01'),(425,20250929103528,1,'2020-01-01 01:01:01'),(426,20251003094629,1,'2020-01-01 01:01:01'),(427,20251009091733,1,'2020-01-01 01:01:01'); +INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'),(120,20220201084510,1,'2020-01-01 01:01:01'),(121,20220208144830,1,'2020-01-01 01:01:01'),(122,20220208144831,1,'2020-01-01 01:01:01'),(123,20220215152203,1,'2020-01-01 01:01:01'),(124,20220223113157,1,'2020-01-01 01:01:01'),(125,20220307104655,1,'2020-01-01 01:01:01'),(126,20220309133956,1,'2020-01-01 01:01:01'),(127,20220316155700,1,'2020-01-01 01:01:01'),(128,20220323152301,1,'2020-01-01 01:01:01'),(129,20220330100659,1,'2020-01-01 01:01:01'),(130,20220404091216,1,'2020-01-01 01:01:01'),(131,20220419140750,1,'2020-01-01 01:01:01'),(132,20220428140039,1,'2020-01-01 01:01:01'),(133,20220503134048,1,'2020-01-01 01:01:01'),(134,20220524102918,1,'2020-01-01 01:01:01'),(135,20220526123327,1,'2020-01-01 01:01:01'),(136,20220526123328,1,'2020-01-01 01:01:01'),(137,20220526123329,1,'2020-01-01 01:01:01'),(138,20220608113128,1,'2020-01-01 01:01:01'),(139,20220627104817,1,'2020-01-01 01:01:01'),(140,20220704101843,1,'2020-01-01 01:01:01'),(141,20220708095046,1,'2020-01-01 01:01:01'),(142,20220713091130,1,'2020-01-01 01:01:01'),(143,20220802135510,1,'2020-01-01 01:01:01'),(144,20220818101352,1,'2020-01-01 01:01:01'),(145,20220822161445,1,'2020-01-01 01:01:01'),(146,20220831100036,1,'2020-01-01 01:01:01'),(147,20220831100151,1,'2020-01-01 01:01:01'),(148,20220908181826,1,'2020-01-01 01:01:01'),(149,20220914154915,1,'2020-01-01 01:01:01'),(150,20220915165115,1,'2020-01-01 01:01:01'),(151,20220915165116,1,'2020-01-01 01:01:01'),(152,20220928100158,1,'2020-01-01 01:01:01'),(153,20221014084130,1,'2020-01-01 01:01:01'),(154,20221027085019,1,'2020-01-01 01:01:01'),(155,20221101103952,1,'2020-01-01 01:01:01'),(156,20221104144401,1,'2020-01-01 01:01:01'),(157,20221109100749,1,'2020-01-01 01:01:01'),(158,20221115104546,1,'2020-01-01 01:01:01'),(159,20221130114928,1,'2020-01-01 01:01:01'),(160,20221205112142,1,'2020-01-01 01:01:01'),(161,20221216115820,1,'2020-01-01 01:01:01'),(162,20221220195934,1,'2020-01-01 01:01:01'),(163,20221220195935,1,'2020-01-01 01:01:01'),(164,20221223174807,1,'2020-01-01 01:01:01'),(165,20221227163855,1,'2020-01-01 01:01:01'),(166,20221227163856,1,'2020-01-01 01:01:01'),(167,20230202224725,1,'2020-01-01 01:01:01'),(168,20230206163608,1,'2020-01-01 01:01:01'),(169,20230214131519,1,'2020-01-01 01:01:01'),(170,20230303135738,1,'2020-01-01 01:01:01'),(171,20230313135301,1,'2020-01-01 01:01:01'),(172,20230313141819,1,'2020-01-01 01:01:01'),(173,20230315104937,1,'2020-01-01 01:01:01'),(174,20230317173844,1,'2020-01-01 01:01:01'),(175,20230320133602,1,'2020-01-01 01:01:01'),(176,20230330100011,1,'2020-01-01 01:01:01'),(177,20230330134823,1,'2020-01-01 01:01:01'),(178,20230405232025,1,'2020-01-01 01:01:01'),(179,20230408084104,1,'2020-01-01 01:01:01'),(180,20230411102858,1,'2020-01-01 01:01:01'),(181,20230421155932,1,'2020-01-01 01:01:01'),(182,20230425082126,1,'2020-01-01 01:01:01'),(183,20230425105727,1,'2020-01-01 01:01:01'),(184,20230501154913,1,'2020-01-01 01:01:01'),(185,20230503101418,1,'2020-01-01 01:01:01'),(186,20230515144206,1,'2020-01-01 01:01:01'),(187,20230517140952,1,'2020-01-01 01:01:01'),(188,20230517152807,1,'2020-01-01 01:01:01'),(189,20230518114155,1,'2020-01-01 01:01:01'),(190,20230520153236,1,'2020-01-01 01:01:01'),(191,20230525151159,1,'2020-01-01 01:01:01'),(192,20230530122103,1,'2020-01-01 01:01:01'),(193,20230602111827,1,'2020-01-01 01:01:01'),(194,20230608103123,1,'2020-01-01 01:01:01'),(195,20230629140529,1,'2020-01-01 01:01:01'),(196,20230629140530,1,'2020-01-01 01:01:01'),(197,20230711144622,1,'2020-01-01 01:01:01'),(198,20230721135421,1,'2020-01-01 01:01:01'),(199,20230721161508,1,'2020-01-01 01:01:01'),(200,20230726115701,1,'2020-01-01 01:01:01'),(201,20230807100822,1,'2020-01-01 01:01:01'),(202,20230814150442,1,'2020-01-01 01:01:01'),(203,20230823122728,1,'2020-01-01 01:01:01'),(204,20230906152143,1,'2020-01-01 01:01:01'),(205,20230911163618,1,'2020-01-01 01:01:01'),(206,20230912101759,1,'2020-01-01 01:01:01'),(207,20230915101341,1,'2020-01-01 01:01:01'),(208,20230918132351,1,'2020-01-01 01:01:01'),(209,20231004144339,1,'2020-01-01 01:01:01'),(210,20231009094541,1,'2020-01-01 01:01:01'),(211,20231009094542,1,'2020-01-01 01:01:01'),(212,20231009094543,1,'2020-01-01 01:01:01'),(213,20231009094544,1,'2020-01-01 01:01:01'),(214,20231016091915,1,'2020-01-01 01:01:01'),(215,20231024174135,1,'2020-01-01 01:01:01'),(216,20231025120016,1,'2020-01-01 01:01:01'),(217,20231025160156,1,'2020-01-01 01:01:01'),(218,20231031165350,1,'2020-01-01 01:01:01'),(219,20231106144110,1,'2020-01-01 01:01:01'),(220,20231107130934,1,'2020-01-01 01:01:01'),(221,20231109115838,1,'2020-01-01 01:01:01'),(222,20231121054530,1,'2020-01-01 01:01:01'),(223,20231122101320,1,'2020-01-01 01:01:01'),(224,20231130132828,1,'2020-01-01 01:01:01'),(225,20231130132931,1,'2020-01-01 01:01:01'),(226,20231204155427,1,'2020-01-01 01:01:01'),(227,20231206142340,1,'2020-01-01 01:01:01'),(228,20231207102320,1,'2020-01-01 01:01:01'),(229,20231207102321,1,'2020-01-01 01:01:01'),(230,20231207133731,1,'2020-01-01 01:01:01'),(231,20231212094238,1,'2020-01-01 01:01:01'),(232,20231212095734,1,'2020-01-01 01:01:01'),(233,20231212161121,1,'2020-01-01 01:01:01'),(234,20231215122713,1,'2020-01-01 01:01:01'),(235,20231219143041,1,'2020-01-01 01:01:01'),(236,20231224070653,1,'2020-01-01 01:01:01'),(237,20240110134315,1,'2020-01-01 01:01:01'),(238,20240119091637,1,'2020-01-01 01:01:01'),(239,20240126020642,1,'2020-01-01 01:01:01'),(240,20240126020643,1,'2020-01-01 01:01:01'),(241,20240129162819,1,'2020-01-01 01:01:01'),(242,20240130115133,1,'2020-01-01 01:01:01'),(243,20240131083822,1,'2020-01-01 01:01:01'),(244,20240205095928,1,'2020-01-01 01:01:01'),(245,20240205121956,1,'2020-01-01 01:01:01'),(246,20240209110212,1,'2020-01-01 01:01:01'),(247,20240212111533,1,'2020-01-01 01:01:01'),(248,20240221112844,1,'2020-01-01 01:01:01'),(249,20240222073518,1,'2020-01-01 01:01:01'),(250,20240222135115,1,'2020-01-01 01:01:01'),(251,20240226082255,1,'2020-01-01 01:01:01'),(252,20240228082706,1,'2020-01-01 01:01:01'),(253,20240301173035,1,'2020-01-01 01:01:01'),(254,20240302111134,1,'2020-01-01 01:01:01'),(255,20240312103753,1,'2020-01-01 01:01:01'),(256,20240313143416,1,'2020-01-01 01:01:01'),(257,20240314085226,1,'2020-01-01 01:01:01'),(258,20240314151747,1,'2020-01-01 01:01:01'),(259,20240320145650,1,'2020-01-01 01:01:01'),(260,20240327115530,1,'2020-01-01 01:01:01'),(261,20240327115617,1,'2020-01-01 01:01:01'),(262,20240408085837,1,'2020-01-01 01:01:01'),(263,20240415104633,1,'2020-01-01 01:01:01'),(264,20240430111727,1,'2020-01-01 01:01:01'),(265,20240515200020,1,'2020-01-01 01:01:01'),(266,20240521143023,1,'2020-01-01 01:01:01'),(267,20240521143024,1,'2020-01-01 01:01:01'),(268,20240601174138,1,'2020-01-01 01:01:01'),(269,20240607133721,1,'2020-01-01 01:01:01'),(270,20240612150059,1,'2020-01-01 01:01:01'),(271,20240613162201,1,'2020-01-01 01:01:01'),(272,20240613172616,1,'2020-01-01 01:01:01'),(273,20240618142419,1,'2020-01-01 01:01:01'),(274,20240625093543,1,'2020-01-01 01:01:01'),(275,20240626195531,1,'2020-01-01 01:01:01'),(276,20240702123921,1,'2020-01-01 01:01:01'),(277,20240703154849,1,'2020-01-01 01:01:01'),(278,20240707134035,1,'2020-01-01 01:01:01'),(279,20240707134036,1,'2020-01-01 01:01:01'),(280,20240709124958,1,'2020-01-01 01:01:01'),(281,20240709132642,1,'2020-01-01 01:01:01'),(282,20240709183940,1,'2020-01-01 01:01:01'),(283,20240710155623,1,'2020-01-01 01:01:01'),(284,20240723102712,1,'2020-01-01 01:01:01'),(285,20240725152735,1,'2020-01-01 01:01:01'),(286,20240725182118,1,'2020-01-01 01:01:01'),(287,20240726100517,1,'2020-01-01 01:01:01'),(288,20240730171504,1,'2020-01-01 01:01:01'),(289,20240730174056,1,'2020-01-01 01:01:01'),(290,20240730215453,1,'2020-01-01 01:01:01'),(291,20240730374423,1,'2020-01-01 01:01:01'),(292,20240801115359,1,'2020-01-01 01:01:01'),(293,20240802101043,1,'2020-01-01 01:01:01'),(294,20240802113716,1,'2020-01-01 01:01:01'),(295,20240814135330,1,'2020-01-01 01:01:01'),(296,20240815000000,1,'2020-01-01 01:01:01'),(297,20240815000001,1,'2020-01-01 01:01:01'),(298,20240816103247,1,'2020-01-01 01:01:01'),(299,20240820091218,1,'2020-01-01 01:01:01'),(300,20240826111228,1,'2020-01-01 01:01:01'),(301,20240826160025,1,'2020-01-01 01:01:01'),(302,20240829165448,1,'2020-01-01 01:01:01'),(303,20240829165605,1,'2020-01-01 01:01:01'),(304,20240829165715,1,'2020-01-01 01:01:01'),(305,20240829165930,1,'2020-01-01 01:01:01'),(306,20240829170023,1,'2020-01-01 01:01:01'),(307,20240829170033,1,'2020-01-01 01:01:01'),(308,20240829170044,1,'2020-01-01 01:01:01'),(309,20240905105135,1,'2020-01-01 01:01:01'),(310,20240905140514,1,'2020-01-01 01:01:01'),(311,20240905200000,1,'2020-01-01 01:01:01'),(312,20240905200001,1,'2020-01-01 01:01:01'),(313,20241002104104,1,'2020-01-01 01:01:01'),(314,20241002104105,1,'2020-01-01 01:01:01'),(315,20241002104106,1,'2020-01-01 01:01:01'),(316,20241002210000,1,'2020-01-01 01:01:01'),(317,20241003145349,1,'2020-01-01 01:01:01'),(318,20241004005000,1,'2020-01-01 01:01:01'),(319,20241008083925,1,'2020-01-01 01:01:01'),(320,20241009090010,1,'2020-01-01 01:01:01'),(321,20241017163402,1,'2020-01-01 01:01:01'),(322,20241021224359,1,'2020-01-01 01:01:01'),(323,20241022140321,1,'2020-01-01 01:01:01'),(324,20241025111236,1,'2020-01-01 01:01:01'),(325,20241025112748,1,'2020-01-01 01:01:01'),(326,20241025141855,1,'2020-01-01 01:01:01'),(327,20241110152839,1,'2020-01-01 01:01:01'),(328,20241110152840,1,'2020-01-01 01:01:01'),(329,20241110152841,1,'2020-01-01 01:01:01'),(330,20241116233322,1,'2020-01-01 01:01:01'),(331,20241122171434,1,'2020-01-01 01:01:01'),(332,20241125150614,1,'2020-01-01 01:01:01'),(333,20241203125346,1,'2020-01-01 01:01:01'),(334,20241203130032,1,'2020-01-01 01:01:01'),(335,20241205122800,1,'2020-01-01 01:01:01'),(336,20241209164540,1,'2020-01-01 01:01:01'),(337,20241210140021,1,'2020-01-01 01:01:01'),(338,20241219180042,1,'2020-01-01 01:01:01'),(339,20241220100000,1,'2020-01-01 01:01:01'),(340,20241220114903,1,'2020-01-01 01:01:01'),(341,20241220114904,1,'2020-01-01 01:01:01'),(342,20241224000000,1,'2020-01-01 01:01:01'),(343,20241230000000,1,'2020-01-01 01:01:01'),(344,20241231112624,1,'2020-01-01 01:01:01'),(345,20250102121439,1,'2020-01-01 01:01:01'),(346,20250121094045,1,'2020-01-01 01:01:01'),(347,20250121094500,1,'2020-01-01 01:01:01'),(348,20250121094600,1,'2020-01-01 01:01:01'),(349,20250121094700,1,'2020-01-01 01:01:01'),(350,20250124194347,1,'2020-01-01 01:01:01'),(351,20250127162751,1,'2020-01-01 01:01:01'),(352,20250213104005,1,'2020-01-01 01:01:01'),(353,20250214205657,1,'2020-01-01 01:01:01'),(354,20250217093329,1,'2020-01-01 01:01:01'),(355,20250219090511,1,'2020-01-01 01:01:01'),(356,20250219100000,1,'2020-01-01 01:01:01'),(357,20250219142401,1,'2020-01-01 01:01:01'),(358,20250224184002,1,'2020-01-01 01:01:01'),(359,20250225085436,1,'2020-01-01 01:01:01'),(360,20250226000000,1,'2020-01-01 01:01:01'),(361,20250226153445,1,'2020-01-01 01:01:01'),(362,20250304162702,1,'2020-01-01 01:01:01'),(363,20250306144233,1,'2020-01-01 01:01:01'),(364,20250313163430,1,'2020-01-01 01:01:01'),(365,20250317130944,1,'2020-01-01 01:01:01'),(366,20250318165922,1,'2020-01-01 01:01:01'),(367,20250320132525,1,'2020-01-01 01:01:01'),(368,20250320200000,1,'2020-01-01 01:01:01'),(369,20250326161930,1,'2020-01-01 01:01:01'),(370,20250326161931,1,'2020-01-01 01:01:01'),(371,20250331042354,1,'2020-01-01 01:01:01'),(372,20250331154206,1,'2020-01-01 01:01:01'),(373,20250401155831,1,'2020-01-01 01:01:01'),(374,20250408133233,1,'2020-01-01 01:01:01'),(375,20250410104321,1,'2020-01-01 01:01:01'),(376,20250421085116,1,'2020-01-01 01:01:01'),(377,20250422095806,1,'2020-01-01 01:01:01'),(378,20250424153059,1,'2020-01-01 01:01:01'),(379,20250430103833,1,'2020-01-01 01:01:01'),(380,20250430112622,1,'2020-01-01 01:01:01'),(381,20250501162727,1,'2020-01-01 01:01:01'),(382,20250502154517,1,'2020-01-01 01:01:01'),(383,20250502222222,1,'2020-01-01 01:01:01'),(384,20250507170845,1,'2020-01-01 01:01:01'),(385,20250513162912,1,'2020-01-01 01:01:01'),(386,20250519161614,1,'2020-01-01 01:01:01'),(387,20250519170000,1,'2020-01-01 01:01:01'),(388,20250520153848,1,'2020-01-01 01:01:01'),(389,20250528115932,1,'2020-01-01 01:01:01'),(390,20250529102706,1,'2020-01-01 01:01:01'),(391,20250603105558,1,'2020-01-01 01:01:01'),(392,20250609102714,1,'2020-01-01 01:01:01'),(393,20250609112613,1,'2020-01-01 01:01:01'),(394,20250613103810,1,'2020-01-01 01:01:01'),(395,20250616193950,1,'2020-01-01 01:01:01'),(396,20250624140757,1,'2020-01-01 01:01:01'),(397,20250626130239,1,'2020-01-01 01:01:01'),(398,20250629131032,1,'2020-01-01 01:01:01'),(399,20250701155654,1,'2020-01-01 01:01:01'),(400,20250707095725,1,'2020-01-01 01:01:01'),(401,20250716152435,1,'2020-01-01 01:01:01'),(402,20250718091828,1,'2020-01-01 01:01:01'),(403,20250728122229,1,'2020-01-01 01:01:01'),(404,20250731122715,1,'2020-01-01 01:01:01'),(405,20250731151000,1,'2020-01-01 01:01:01'),(406,20250803000000,1,'2020-01-01 01:01:01'),(407,20250805083116,1,'2020-01-01 01:01:01'),(408,20250807140441,1,'2020-01-01 01:01:01'),(409,20250808000000,1,'2020-01-01 01:01:01'),(410,20250811155036,1,'2020-01-01 01:01:01'),(411,20250813205039,1,'2020-01-01 01:01:01'),(412,20250814123333,1,'2020-01-01 01:01:01'),(413,20250815130115,1,'2020-01-01 01:01:01'),(414,20250816115553,1,'2020-01-01 01:01:01'),(415,20250817154557,1,'2020-01-01 01:01:01'),(416,20250825113751,1,'2020-01-01 01:01:01'),(417,20250827113140,1,'2020-01-01 01:01:01'),(418,20250828120836,1,'2020-01-01 01:01:01'),(419,20250902112642,1,'2020-01-01 01:01:01'),(420,20250904091745,1,'2020-01-01 01:01:01'),(421,20250905090000,1,'2020-01-01 01:01:01'),(422,20250922083056,1,'2020-01-01 01:01:01'),(423,20250923120000,1,'2020-01-01 01:01:01'),(424,20250926123048,1,'2020-01-01 01:01:01'),(425,20250929103528,1,'2020-01-01 01:01:01'),(426,20251003094629,1,'2020-01-01 01:01:01'),(427,20251009091733,1,'2020-01-01 01:01:01'),(428,20251010153829,1,'2020-01-01 01:01:01'); /*!40101 SET @saved_cs_client = @@character_set_client */; /*!50503 SET character_set_client = utf8mb4 */; CREATE TABLE `mobile_device_management_solutions` ( diff --git a/server/datastore/mysql/software.go b/server/datastore/mysql/software.go index 722cea02e5d7..6d5587e14fb7 100644 --- a/server/datastore/mysql/software.go +++ b/server/datastore/mysql/software.go @@ -6,6 +6,7 @@ import ( "encoding/hex" "errors" "fmt" + "regexp" "sort" "strconv" "strings" @@ -43,6 +44,9 @@ var tracer = otel.Tracer("github.com/fleetdm/fleet/v4/server/datastore/mysql") // This is a variable so it can be adjusted during unit testing. var countHostSoftwareBatchSize = uint64(100000) +// trailingNonWordChars matches trailing anything not a letter, digit, or underscore +var trailingNonWordChars = regexp.MustCompile(`\W+$`) + // Since a host may have a lot of software items, we need to batch the inserts. // The maximum number of software items we can insert at one time is governed by max_allowed_packet, which already be set to a high value for MDM bootstrap packages, // and by the maximum number of placeholders in a prepared statement, which is 65,536. These are already fairly large limits. @@ -404,7 +408,7 @@ func (ds *Datastore) applyChangesForNewSoftwareDB( return r, nil } - existingSoftware, incomingByChecksum, existingTitlesForNewSoftware, existingBundleIDsToUpdate, err := ds.getExistingSoftware(ctx, current, incoming) + existingSoftware, incomingByChecksum, existingTitlesForNewSoftware, err := ds.getExistingSoftware(ctx, current, incoming) if err != nil { return r, err } @@ -438,54 +442,11 @@ func (ds *Datastore) applyChangesForNewSoftwareDB( } r.Inserted = inserted - // Also link existing software that matches by bundle ID - // This handles the case where software has the same bundle ID but different name. - // Since this is a rare case, it is not optimized for performance. - if len(existingBundleIDsToUpdate) > 0 { - bundleInserted, err := ds.linkExistingBundleIDSoftware(ctx, tx, hostID, existingBundleIDsToUpdate) - if err != nil { - return err - } - r.Inserted = append(r.Inserted, bundleInserted...) - - // Build map of software IDs to their new names for renaming. - // softwareRenames should match existingBundleIDsToUpdate, but we create this extra map to handle - // software entries that share the same bundle ID as well as other potential corner cases. - softwareRenames := make(map[uint]string, len(existingBundleIDsToUpdate)) - // Check inserted software for renames - for _, sw := range r.Inserted { - if sw.BundleIdentifier != "" { - if updSoftwareList, needsUpdate := existingBundleIDsToUpdate[sw.BundleIdentifier]; needsUpdate { - // Use the first software in the list for the name (they should all have the same name) - if len(updSoftwareList) > 0 { - softwareRenames[sw.ID] = updSoftwareList[0].Name - } - } - } - } - // Check existing software for renames - for _, s := range existingSoftware { - if s.BundleIdentifier != nil && *s.BundleIdentifier != "" { - if updSoftwareList, ok := existingBundleIDsToUpdate[*s.BundleIdentifier]; ok { - // Use the first software in the list for the name (they should all have the same name) - if len(updSoftwareList) > 0 { - softwareRenames[s.ID] = updSoftwareList[0].Name - } - } - } - } - - if err = updateTargetedBundleIDs(ctx, tx, softwareRenames); err != nil { - return err - } - } - - // Use r.Inserted which contains all inserted items (including bundle ID matches) if err = checkForDeletedInstalledSoftware(ctx, tx, deleted, r.Inserted, hostID); err != nil { return err } - if err = updateModifiedHostSoftwareDB(ctx, tx, hostID, current, incoming, existingBundleIDsToUpdate, ds.minLastOpenedAtDiff, ds.logger); err != nil { + if err = updateModifiedHostSoftwareDB(ctx, tx, hostID, current, incoming, ds.minLastOpenedAtDiff, ds.logger); err != nil { return err } @@ -501,108 +462,6 @@ func (ds *Datastore) applyChangesForNewSoftwareDB( return r, err } -// updateTargetedBundleIDs updates software names when bundle IDs match but names differ. -// softwareRenames maps software IDs to their new names. -func updateTargetedBundleIDs(ctx context.Context, tx sqlx.ExtContext, softwareRenames map[uint]string) error { - if len(softwareRenames) == 0 { - return nil - } - - // Extract software IDs for batch processing - softwareIDs := make([]uint, 0, len(softwareRenames)) - for id := range softwareRenames { - softwareIDs = append(softwareIDs, id) - } - - const batchSize = 100 - - err := common_mysql.BatchProcessSimple(softwareIDs, batchSize, func(batch []uint) error { - placeholders := make([]string, len(batch)) - args := make([]any, len(batch)) - for i, id := range batch { - placeholders[i] = "?" - args[i] = id - } - - // During high concurrency situations, we may have multiple transactions attempting to update the same software rows. - // For example, this can happen when many hosts are trying to rename the same software items. - // To avoid long locks or even deadlocks, use UPDATE SKIP LOCKED to skip rows that are already locked by another transaction. - // This means that some software rows may not be updated in this transaction, - // however, eventually they should be updated. This is trading off immediate consistency - // for less lock contention. - lockQuery := fmt.Sprintf( - "SELECT id, name FROM software WHERE id IN (%s) ORDER BY id FOR UPDATE SKIP LOCKED", - strings.Join(placeholders, ","), - ) - - rows, err := tx.QueryContext(ctx, lockQuery, args...) - if err != nil { - return ctxerr.Wrap(ctx, err, "lock software rows for rename") - } - defer rows.Close() - - type lockedRow struct { - id uint - currentName string - } - var lockedRows []lockedRow - for rows.Next() { - var lr lockedRow - if err := rows.Scan(&lr.id, &lr.currentName); err != nil { - return ctxerr.Wrap(ctx, err, "scan locked row") - } - lockedRows = append(lockedRows, lr) - } - if err := rows.Err(); err != nil { - return ctxerr.Wrap(ctx, err, "iterate locked rows") - } - - if len(lockedRows) == 0 { - return nil - } - - var rowsToUpdate []lockedRow - for _, lr := range lockedRows { - newName := softwareRenames[lr.id] - if lr.currentName != newName { - rowsToUpdate = append(rowsToUpdate, lr) - } - } - - if len(rowsToUpdate) == 0 { - return nil - } - - updateCases := make([]string, 0, len(rowsToUpdate)) - updateCaseArgs := make([]any, 0, len(rowsToUpdate)*2) - updateWhereArgs := make([]any, 0, len(rowsToUpdate)) - updateIDs := make([]string, 0, len(rowsToUpdate)) - - for _, lr := range rowsToUpdate { - newName := softwareRenames[lr.id] - updateCases = append(updateCases, "WHEN ? THEN ?") - updateCaseArgs = append(updateCaseArgs, lr.id, newName) - updateWhereArgs = append(updateWhereArgs, lr.id) - updateIDs = append(updateIDs, "?") - } - - updateStmt := fmt.Sprintf( - `UPDATE software SET name = CASE id %s END, name_source = 'bundle_4.67' WHERE id IN (%s)`, - strings.Join(updateCases, " "), - strings.Join(updateIDs, ","), - ) - - _, err = tx.ExecContext(ctx, updateStmt, append(updateCaseArgs, updateWhereArgs...)...) - if err != nil { - return ctxerr.Wrap(ctx, err, "batch update software names") - } - - return nil - }) - - return err -} - func checkForDeletedInstalledSoftware(ctx context.Context, tx sqlx.ExtContext, deleted []fleet.Software, inserted []fleet.Software, hostID uint, ) error { @@ -677,27 +536,20 @@ func (ds *Datastore) getExistingSoftware( currentSoftware []softwareIDChecksum, incomingChecksumToSoftware map[string]fleet.Software, incomingChecksumToTitle map[string]fleet.SoftwareTitle, - existingBundleIDsToUpdate map[string][]fleet.Software, err error, ) { // Compute checksums for all incoming software, which we will use for faster retrieval, since checksum is a unique index incomingChecksumToSoftware = make(map[string]fleet.Software, len(current)) newSoftware := make(map[string]struct{}) - incomingBundleIDsToNewSoftwareNames := make(map[string]string) - existingBundleIDsToUpdate = make(map[string][]fleet.Software) for uniqueName, s := range incoming { _, ok := current[uniqueName] if !ok { checksum, err := s.ComputeRawChecksum() if err != nil { - return nil, nil, nil, nil, err + return nil, nil, nil, err } incomingChecksumToSoftware[string(checksum)] = s newSoftware[string(checksum)] = struct{}{} - - if s.BundleIdentifier != "" { - incomingBundleIDsToNewSoftwareNames[s.BundleIdentifier] = s.Name - } } } @@ -710,45 +562,32 @@ func (ds *Datastore) getExistingSoftware( // It is OK if the software is not found in the replica DB, because we will then attempt to insert it in the master DB. currentSoftware, err = getSoftwareIDsByChecksums(ctx, ds.reader(ctx), keys) if err != nil { - return nil, nil, nil, nil, err + return nil, nil, nil, err } for _, currentSoftwareItem := range currentSoftware { - incomingSoftwareItem, ok := incomingChecksumToSoftware[currentSoftwareItem.Checksum] + _, ok := incomingChecksumToSoftware[currentSoftwareItem.Checksum] if !ok { // This should never happen. If it does, we have a bug. - return nil, nil, nil, nil, ctxerr.New( + return nil, nil, nil, ctxerr.New( ctx, fmt.Sprintf("current software: software not found for checksum %s", hex.EncodeToString([]byte(currentSoftwareItem.Checksum))), ) } - if currentSoftwareItem.BundleIdentifier != nil && currentSoftwareItem.Source == "apps" { - if name, ok := incomingBundleIDsToNewSoftwareNames[*currentSoftwareItem.BundleIdentifier]; ok && name != currentSoftwareItem.Name { - // Then this is a software whose name has changed, so we should update the name - // Copy the incoming software but with the existing software's ID - swWithID := incomingSoftwareItem - swWithID.ID = currentSoftwareItem.ID - existingBundleIDsToUpdate[*currentSoftwareItem.BundleIdentifier] = append(existingBundleIDsToUpdate[*currentSoftwareItem.BundleIdentifier], swWithID) - - // Delete this checksum to prevent it from being treated as new software - delete(incomingChecksumToSoftware, currentSoftwareItem.Checksum) - continue - } - } delete(newSoftware, currentSoftwareItem.Checksum) } } if len(newSoftware) == 0 { - return currentSoftware, incomingChecksumToSoftware, incomingChecksumToTitle, existingBundleIDsToUpdate, nil + return currentSoftware, incomingChecksumToSoftware, incomingChecksumToTitle, nil } // There's new software, so we try to get the titles already stored in `software_titles` for them. incomingChecksumToTitle, _, err = ds.getIncomingSoftwareChecksumsToExistingTitles(ctx, newSoftware, incomingChecksumToSoftware) if err != nil { - return nil, nil, nil, nil, ctxerr.Wrap(ctx, err, "get incoming software checksums to existing titles") + return nil, nil, nil, ctxerr.Wrap(ctx, err, "get incoming software checksums to existing titles") } - return currentSoftware, incomingChecksumToSoftware, incomingChecksumToTitle, existingBundleIDsToUpdate, nil + return currentSoftware, incomingChecksumToSoftware, incomingChecksumToTitle, nil } // getIncomingSoftwareChecksumsToExistingTitles loads the existing titles for the new incoming software. @@ -913,6 +752,33 @@ func deleteUninstalledHostSoftwareDB( return deletedSoftware, nil } +// longestCommonPrefix finds the longest common prefix among a slice of strings. +// Returns empty string if there's no common prefix. +func longestCommonPrefix(strs []string) string { + if len(strs) == 0 { + return "" + } + if len(strs) == 1 { + return strs[0] + } + + firstLen := len(strs[0]) + i := 0 + for { + if i >= firstLen { + return strs[0] + } + + c := strs[0][i] + for _, s := range strs[1:] { + if i >= len(s) || s[i] != c { + return strs[0][:i] + } + } + i++ + } +} + // preInsertSoftwareInventory pre-inserts software and software_titles outside the main transaction // to reduce lock contention. These operations are idempotent due to INSERT IGNORE. func (ds *Datastore) preInsertSoftwareInventory( @@ -921,15 +787,37 @@ func (ds *Datastore) preInsertSoftwareInventory( softwareChecksums map[string]fleet.Software, existingTitlesForNewSoftware map[string]fleet.SoftwareTitle, ) error { + type titleKey struct { + name string + source string + extensionFor string + bundleID string + isKernel bool + } + // Collect all software that needs to be inserted needsInsert := make(map[string]fleet.Software) + bundleGroups := make(map[titleKey][]string) + keys := make([]string, 0, len(softwareChecksums)) + existingSet := make(map[string]struct{}, len(existingSoftware)) for _, es := range existingSoftware { existingSet[es.Checksum] = struct{}{} } + for checksum, sw := range softwareChecksums { if _, ok := existingSet[checksum]; !ok { needsInsert[checksum] = sw + keys = append(keys, checksum) + + if sw.BundleIdentifier != "" { + key := titleKey{ + bundleID: sw.BundleIdentifier, + source: sw.Source, + extensionFor: sw.ExtensionFor, + } + bundleGroups[key] = append(bundleGroups[key], sw.Name) + } } } @@ -937,12 +825,31 @@ func (ds *Datastore) preInsertSoftwareInventory( return nil } - // Process in smaller batches to reduce lock time - keys := make([]string, 0, len(needsInsert)) - for checksum := range needsInsert { - keys = append(keys, checksum) + bestTitleNames := make(map[titleKey]string) + for key, names := range bundleGroups { + if len(names) > 1 { + // Pick the best represenative name for the group of names + commonPrefix := longestCommonPrefix(names) + commonPrefix = trailingNonWordChars.ReplaceAllString(commonPrefix, "") + if len(commonPrefix) > 0 { + bestTitleNames[key] = commonPrefix + } else { + // Fall back to shortest name + shortest := names[0] + for _, name := range names[1:] { + if len(name) < len(shortest) { + shortest = name + } + } + bestTitleNames[key] = shortest + } + } else if len(names) == 1 { + // Single title or no bundle_identifier + bestTitleNames[key] = names[0] + } } + // Process in smaller batches to reduce lock time err := common_mysql.BatchProcessSimple(keys, softwareInventoryInsertBatchSize, func(batchKeys []string) error { batchSoftware := make(map[string]fleet.Software, len(batchKeys)) for _, key := range batchKeys { @@ -955,8 +862,20 @@ func (ds *Datastore) preInsertSoftwareInventory( newTitlesNeeded := make(map[string]fleet.SoftwareTitle) for checksum, sw := range batchSoftware { if _, ok := existingTitlesForNewSoftware[checksum]; !ok { + titleName := sw.Name + if sw.BundleIdentifier != "" { + key := titleKey{ + bundleID: sw.BundleIdentifier, + source: sw.Source, + extensionFor: sw.ExtensionFor, + } + if computedName, exists := bestTitleNames[key]; exists { + titleName = computedName + } + } + st := fleet.SoftwareTitle{ - Name: sw.Name, + Name: titleName, Source: sw.Source, ExtensionFor: sw.ExtensionFor, IsKernel: sw.IsKernel, @@ -980,15 +899,7 @@ func (ds *Datastore) preInsertSoftwareInventory( } if len(newTitlesNeeded) > 0 { - // Deduplicate titles before insertion to avoid unnecessary duplicate INSERTs - type titleKey struct { - name string - source string - extensionFor string - bundleID string - isKernel bool - } - uniqueTitlesToInsert := make(map[titleKey]fleet.SoftwareTitle, len(newTitlesNeeded)) + uniqueTitlesToInsert := make(map[titleKey]fleet.SoftwareTitle) for _, title := range newTitlesNeeded { bundleID := "" if title.BundleIdentifier != nil { @@ -1001,11 +912,14 @@ func (ds *Datastore) preInsertSoftwareInventory( bundleID: bundleID, isKernel: title.IsKernel, } - uniqueTitlesToInsert[key] = title + + if _, exists := uniqueTitlesToInsert[key]; !exists { + uniqueTitlesToInsert[key] = title + } } // Insert software titles - const numberOfArgsPerSoftwareTitles = 5 + const numberOfArgsPerSoftwareTitles = 6 titlesValues := strings.TrimSuffix(strings.Repeat("(?,?,?,?,?,?),", len(uniqueTitlesToInsert)), ",") titlesStmt := fmt.Sprintf("INSERT IGNORE INTO software_titles (name, source, extension_for, bundle_identifier, is_kernel, application_id) VALUES %s", titlesValues) titlesArgs := make([]any, 0, len(uniqueTitlesToInsert)*numberOfArgsPerSoftwareTitles) @@ -1027,20 +941,25 @@ func (ds *Datastore) preInsertSoftwareInventory( BundleIdentifier *string `db:"bundle_identifier"` } - // Build query to retrieve title IDs using the same unique titles we inserted titlePlaceholders := strings.TrimSuffix(strings.Repeat("(?,?,?,?),", len(uniqueTitlesToInsert)), ",") queryArgs := make([]interface{}, 0, len(uniqueTitlesToInsert)*4) for tk := range uniqueTitlesToInsert { + title := uniqueTitlesToInsert[tk] bundleID := "" - if uniqueTitlesToInsert[tk].BundleIdentifier != nil { - bundleID = *uniqueTitlesToInsert[tk].BundleIdentifier + if title.BundleIdentifier != nil { + bundleID = *title.BundleIdentifier + } + + firstArg := title.Name + if bundleID != "" { + firstArg = bundleID } - queryArgs = append(queryArgs, tk.name, tk.source, tk.extensionFor, bundleID) + queryArgs = append(queryArgs, firstArg, title.Source, title.ExtensionFor, bundleID) } queryTitles := fmt.Sprintf(`SELECT id, name, source, extension_for, bundle_identifier FROM software_titles - WHERE (name, source, extension_for, COALESCE(bundle_identifier, '')) IN (%s)`, titlePlaceholders) + WHERE (COALESCE(bundle_identifier, name), source, extension_for, COALESCE(bundle_identifier, '')) IN (%s)`, titlePlaceholders) if err := sqlx.SelectContext(ctx, tx, &titlesData, queryTitles, queryArgs...); err != nil { return ctxerr.Wrap(ctx, err, "select software titles") @@ -1057,7 +976,14 @@ func (ds *Datastore) preInsertSoftwareInventory( if title.BundleIdentifier != nil { titleBundleID = *title.BundleIdentifier } - if td.Name == title.Name && td.Source == title.Source && td.ExtensionFor == title.ExtensionFor && bundleID == titleBundleID { + // For apps with bundle_identifier, match by bundle_identifier (since we may have picked a different name) + // For others, match by name + nameMatches := td.Name == title.Name + if bundleID != "" && titleBundleID != "" { + // Both have bundle_identifier - match by bundle_identifier instead of name + nameMatches = true + } + if nameMatches && td.Source == title.Source && td.ExtensionFor == title.ExtensionFor && bundleID == titleBundleID { titleIDsByChecksum[checksum] = td.ID // Don't break here - multiple checksums can map to the same title // (e.g., when software has same truncated name but different versions (very rare)) @@ -1067,7 +993,7 @@ func (ds *Datastore) preInsertSoftwareInventory( } // Insert software entries - const numberOfArgsPerSoftware = 11 + const numberOfArgsPerSoftware = 12 values := strings.TrimSuffix( strings.Repeat("(?,?,?,?,?,?,?,?,?,?,?,?),", len(batchKeys)), ",", ) @@ -1133,81 +1059,6 @@ func (ds *Datastore) preInsertSoftwareInventory( return err } -// linkExistingBundleIDSoftware links existing software entries that match by bundle ID to the host. -// This handles the case where incoming software has the same bundle ID as existing software but a different name. -func (ds *Datastore) linkExistingBundleIDSoftware( - ctx context.Context, - tx sqlx.ExtContext, - hostID uint, - existingBundleIDsToUpdate map[string][]fleet.Software, -) ([]fleet.Software, error) { - if len(existingBundleIDsToUpdate) == 0 { - return nil, nil - } - - // Collect all software IDs to verify they still exist - softwareIDs := make([]uint, 0, len(existingBundleIDsToUpdate)) - for _, softwareList := range existingBundleIDsToUpdate { - for _, software := range softwareList { - // The software.ID should already be set from getExistingSoftware - if software.ID == 0 { - return nil, ctxerr.New(ctx, "software ID not set for bundle ID match") - } - softwareIDs = append(softwareIDs, software.ID) - } - } - - // Verify software still exists (just like we do in linkSoftwareToHost) - // This prevents creating orphaned references if software was deleted between pre-insertion and now - // This DB call could be removed to squeeze our a little more performance at the risk of orphaned references. - stmt, args, err := sqlx.In(`SELECT id FROM software WHERE id IN (?)`, softwareIDs) - if err != nil { - return nil, ctxerr.Wrap(ctx, err, "build query for existing software verification") - } - - var existingIDs []uint - if err := sqlx.SelectContext(ctx, tx, &existingIDs, stmt, args...); err != nil { - return nil, ctxerr.Wrap(ctx, err, "verify existing bundle ID software") - } - - // Build a set of existing IDs for quick lookup - existingIDSet := make(map[uint]struct{}, len(existingIDs)) - for _, id := range existingIDs { - existingIDSet[id] = struct{}{} - } - - var insertsHostSoftware []any - var insertedSoftware []fleet.Software - - for _, softwareList := range existingBundleIDsToUpdate { - for _, software := range softwareList { - // Only link if software still exists - if _, ok := existingIDSet[software.ID]; ok { - insertsHostSoftware = append(insertsHostSoftware, hostID, software.ID, software.LastOpenedAt) - insertedSoftware = append(insertedSoftware, software) - } else { - // Log missing software but continue - level.Warn(ds.logger).Log( - "msg", "bundle ID software not found after pre-insertion", - "software_id", software.ID, - "name", software.Name, - "bundle_id", software.BundleIdentifier, - ) - } - } - } - - if len(insertsHostSoftware) > 0 { - values := strings.TrimSuffix(strings.Repeat("(?,?,?),", len(insertsHostSoftware)/3), ",") - stmt := fmt.Sprintf(`INSERT IGNORE INTO host_software (host_id, software_id, last_opened_at) VALUES %s`, values) - if _, err := tx.ExecContext(ctx, stmt, insertsHostSoftware...); err != nil { - return nil, ctxerr.Wrap(ctx, err, "link existing bundle ID software") - } - } - - return insertedSoftware, nil -} - // linkSoftwareToHost links pre-inserted software to a host. // This assumes software inventory entries already exist. func (ds *Datastore) linkSoftwareToHost( @@ -1297,7 +1148,6 @@ func updateModifiedHostSoftwareDB( hostID uint, currentMap map[string]fleet.Software, incomingMap map[string]fleet.Software, - existingBundleIDsToUpdate map[string][]fleet.Software, minLastOpenedAtDiff time.Duration, logger log.Logger, ) error { @@ -1308,21 +1158,16 @@ func updateModifiedHostSoftwareDB( if !ok { continue } - // if the new software has no last opened timestamp, we only - // update if the current software has no last opened timestamp - // and is marked as having a name change. + // if the new software has no last opened timestamp, log if the current one did + // (but only for non-apps sources, as apps sources are managed by osquery) if newSw.LastOpenedAt == nil { - if _, ok := existingBundleIDsToUpdate[newSw.BundleIdentifier]; ok && curSw.LastOpenedAt == nil { - keysToUpdate = append(keysToUpdate, key) - } - // Log cases where the new software has no last opened timestamp, the current software does, - // and the software is marked as having a name change. - // This is expected on macOS, but not on windows/linux. - if ok && curSw.LastOpenedAt != nil && newSw.Source != "apps" { - level.Warn(logger).Log( - "msg", "updateModifiedHostSoftwareDB: last opened at is nil for new software, but not for current software", - "new_software", newSw.Name, "current_software", curSw.Name, - "bundle_identifier", newSw.BundleIdentifier, + if curSw.LastOpenedAt != nil && newSw.Source != "apps" { + level.Info(logger).Log( + "msg", "software last_opened_at changed to nil", + "host_id", hostID, + "software_id", curSw.ID, + "software_name", newSw.Name, + "source", newSw.Source, ) } continue diff --git a/server/datastore/mysql/software_test.go b/server/datastore/mysql/software_test.go index 40199e0fa7b3..36b98cf5ab53 100644 --- a/server/datastore/mysql/software_test.go +++ b/server/datastore/mysql/software_test.go @@ -98,6 +98,7 @@ func TestSoftware(t *testing.T) { {"InventoryPendingSoftware", testInventoryPendingSoftware}, {"PreInsertSoftwareInventory", testPreInsertSoftwareInventory}, {"ListHostSoftwareWithExtensionFor", testListHostSoftwareWithExtensionFor}, + {"LongestCommonPrefix", testLongestCommonPrefix}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { @@ -1848,10 +1849,6 @@ func testUpdateHostSoftware(t *testing.T, ds *Datastore) { // When software with the same bundle ID but different name is added, the system // reuses the existing software entry (matched by bundle ID) and links it to the host func testUpdateHostSoftwareSameBundleIDDifferentNames(t *testing.T, ds *Datastore) { - // TEMPORARILY SKIPPED: updateTargetedBundleIDs is commented out for performance reasons - // TODO: Re-enable when updateTargetedBundleIDs is re-enabled - t.Skip("Skipping test: updateTargetedBundleIDs is temporarily disabled for performance reasons") - ctx := t.Context() host := test.NewHost(t, ds, "bundle-host", "", "bundlekey", "bundleuuid", time.Now()) @@ -1867,47 +1864,98 @@ func testUpdateHostSoftwareSameBundleIDDifferentNames(t *testing.T, ds *Datastor require.NoError(t, err) require.Len(t, host.Software, 1) require.Equal(t, "GoLand.app", host.Software[0].Name) - originalSoftwareID := host.Software[0].ID // Now update with the same bundle ID but different name - // The behavior depends on how the system handles bundle ID matching + // Despite having the same bundle id, the software is added with the new name sw = []fleet.Software{ {Name: "GoLand 2.app", Version: "2024.3", Source: "apps", BundleIdentifier: "com.jetbrains.goland"}, } _, err = ds.UpdateHostSoftware(ctx, host.ID, sw) require.NoError(t, err) - // The getExistingSoftware function matches by bundle ID when present, - // so it links to the existing software entry and updates the name err = ds.LoadHostSoftware(ctx, host, false) require.NoError(t, err) require.Len(t, host.Software, 1) - // The existing software entry is reused (matched by bundle ID) and name is updated require.Equal(t, "GoLand 2.app", host.Software[0].Name, "Name should be updated to reflect what's on the host") - require.Equal(t, originalSoftwareID, host.Software[0].ID, "Should reuse the same software row") - - // Verify the name_source was updated - var nameSource string - err = ds.writer(ctx).GetContext(ctx, &nameSource, - `SELECT name_source FROM software WHERE id = ?`, originalSoftwareID) - require.NoError(t, err) - require.Equal(t, "bundle_4.67", nameSource, "Name source should indicate bundle ID match") - // Verify only one software title exists + // Verify only one software title exists (both software entries map to same title by bundle_identifier) var titleCount int err = ds.writer(ctx).GetContext(ctx, &titleCount, `SELECT COUNT(DISTINCT id) FROM software_titles WHERE bundle_identifier = ?`, "com.jetbrains.goland") require.NoError(t, err) - require.Equal(t, 1, titleCount, "Should have only one software title for the bundle ID") + require.Equal(t, 1, titleCount) - // Verify only one software entry exists with this bundle ID - var softwareCount int - err = ds.writer(ctx).GetContext(ctx, &softwareCount, - `SELECT COUNT(DISTINCT id) FROM software WHERE bundle_identifier = ?`, + // Verify two software entries exist with this bundle ID (different names, same bundle_identifier) + var softwareNames []string + err = ds.writer(ctx).SelectContext(ctx, &softwareNames, + `SELECT name FROM software WHERE bundle_identifier = ? ORDER BY name`, "com.jetbrains.goland") require.NoError(t, err) - require.Equal(t, 1, softwareCount, "Should have only one software entry for the bundle ID") + require.Len(t, softwareNames, 2) + require.Equal(t, []string{"GoLand 2.app", "GoLand.app"}, softwareNames) + + // Helper app edge case: + // We have a main app with a name and bundle id + // We have two helper apps with the same bundle id but different name + sw = []fleet.Software{ + {Name: "Postman", Version: "11.60.2", Source: "apps", BundleIdentifier: "com.postmanlabs.mac"}, + {Name: "Postman Helper (GPU)", Version: "", Source: "apps", BundleIdentifier: "com.postmanlabs.mac.helper"}, + {Name: "Postman Helper (Renderer)", Version: "", Source: "apps", BundleIdentifier: "com.postmanlabs.mac.helper"}, + } + _, err = ds.UpdateHostSoftware(ctx, host.ID, sw) + require.NoError(t, err) + + err = ds.LoadHostSoftware(ctx, host, false) + require.NoError(t, err) + require.Len(t, host.Software, 3) + + var softwareRecords []struct { + Name string `db:"name"` + BundleIdentifier string `db:"bundle_identifier"` + } + err = ds.writer(ctx).SelectContext(ctx, &softwareRecords, + `SELECT name, bundle_identifier FROM software WHERE bundle_identifier = ? OR bundle_identifier = ?`, + "com.postmanlabs.mac", "com.postmanlabs.mac.helper") + require.NoError(t, err) + require.Len(t, softwareRecords, 3) + + for _, softwareRecord := range softwareRecords { + switch softwareRecord.Name { + case "Postman": + require.Equal(t, "com.postmanlabs.mac", softwareRecord.BundleIdentifier) + case "Postman Helper (GPU)", "Postman Helper (Renderer)": + require.Equal(t, "com.postmanlabs.mac.helper", softwareRecord.BundleIdentifier) + default: + t.Fatalf("Unexpected software name: %s", softwareRecord.Name) + } + } + + // Re-ingest helper apps with new names + sw = []fleet.Software{ + {Name: "Postman 2", Version: "11.60.2", Source: "apps", BundleIdentifier: "com.postmanlabs.mac"}, + {Name: "Postman Helper 2 (GPU)", Version: "", Source: "apps", BundleIdentifier: "com.postmanlabs.mac.helper"}, + {Name: "Postman Helper 2 (Renderer)", Version: "", Source: "apps", BundleIdentifier: "com.postmanlabs.mac.helper"}, + } + _, err = ds.UpdateHostSoftware(ctx, host.ID, sw) + require.NoError(t, err) + + err = ds.writer(ctx).SelectContext(ctx, &softwareRecords, + `SELECT name, bundle_identifier FROM software WHERE bundle_identifier = ? OR bundle_identifier = ?`, + "com.postmanlabs.mac", "com.postmanlabs.mac.helper") + require.NoError(t, err) + require.Len(t, softwareRecords, 6) + + for _, softwareRecord := range softwareRecords { + switch softwareRecord.Name { + case "Postman", "Postman 2": + require.Equal(t, "com.postmanlabs.mac", softwareRecord.BundleIdentifier) + case "Postman Helper (GPU)", "Postman Helper (Renderer)", "Postman Helper 2 (GPU)", "Postman Helper 2 (Renderer)": + require.Equal(t, "com.postmanlabs.mac.helper", softwareRecord.BundleIdentifier) + default: + t.Fatalf("Unexpected software name: %s", softwareRecord.Name) + } + } } // Test edge case: Software with same name but different bundle identifiers @@ -1946,12 +1994,9 @@ func testUpdateHostSoftwareSameNameDifferentBundleIDs(t *testing.T, ds *Datastor } // Test edge case: Multiple software entries with the same bundle ID -// This validates that bundle ID renaming affects all software with that bundle ID +// This validates that when software with the same bundle ID but different names +// are added from different hosts, we add software entries for each name func testUpdateHostSoftwareMultipleSameBundleID(t *testing.T, ds *Datastore) { - // TEMPORARILY SKIPPED: updateTargetedBundleIDs is commented out for performance reasons - // TODO: Re-enable when updateTargetedBundleIDs is re-enabled - t.Skip("Skipping test: updateTargetedBundleIDs is temporarily disabled for performance reasons") - ctx := t.Context() host1 := test.NewHost(t, ds, "multi-bundle-host1", "", "multikey1", "multiuuid1", time.Now()) host2 := test.NewHost(t, ds, "multi-bundle-host2", "", "multikey2", "multiuuid2", time.Now()) @@ -1979,7 +2024,7 @@ func testUpdateHostSoftwareMultipleSameBundleID(t *testing.T, ds *Datastore) { require.NoError(t, err) // Step 3: Host3 reports the SAME software but with a different name - // This should trigger renaming of ALL software with that bundle ID + // This should not rename all software, it should create a new software entry sw3 := []fleet.Software{ {Name: "GoLand 2024.app", Version: "2024.2", Source: "apps", BundleIdentifier: "com.jetbrains.goland"}, {Name: "GoLand 2024.app", Version: "2024.3-beta", Source: "apps", BundleIdentifier: "com.jetbrains.goland"}, @@ -1987,7 +2032,7 @@ func testUpdateHostSoftwareMultipleSameBundleID(t *testing.T, ds *Datastore) { _, err = ds.UpdateHostSoftware(ctx, host3.ID, sw3) require.NoError(t, err) - // Step 4: Verify the renaming behavior + // Step 4: Verify insertion into software behavior var updatedSoftware []struct { ID uint `db:"id"` Name string `db:"name"` @@ -1999,39 +2044,62 @@ func testUpdateHostSoftwareMultipleSameBundleID(t *testing.T, ds *Datastore) { "com.jetbrains.goland") require.NoError(t, err) - // Should have exactly 2 software entries (one per version) - require.Len(t, updatedSoftware, 2, "Should have exactly 2 software entries (one per version)") + // Should have exactly 4 software entries + require.Len(t, updatedSoftware, 4, "Should have exactly 4 software entries") + + // Verify we have both versions for each name + golandAppVersions := make(map[string]bool) + goland2024AppVersions := make(map[string]bool) - // Both entries should be renamed to "GoLand 2024.app" for _, sw := range updatedSoftware { - t.Logf("Software: ID=%d, Name=%s, Version=%s, NameSource=%s", sw.ID, sw.Name, sw.Version, sw.NameSource) - require.Equal(t, "GoLand 2024.app", sw.Name, "All software with same bundle ID should be renamed") - require.Equal(t, "bundle_4.67", sw.NameSource, "Renamed software should have bundle_4.67 source") + switch sw.Name { + case "GoLand.app": + golandAppVersions[sw.Version] = true + case "GoLand 2024.app": + goland2024AppVersions[sw.Version] = true + default: + t.Fatalf("Unexpected software name: %s", sw.Name) + } } - // Verify that host1 and host2 now see the renamed software + require.Len(t, golandAppVersions, 2, "Should have 2 versions of GoLand.app") + require.True(t, golandAppVersions["2024.2"], "Should have GoLand.app v2024.2") + require.True(t, golandAppVersions["2024.3-beta"], "Should have GoLand.app v2024.3-beta") + + require.Len(t, goland2024AppVersions, 2, "Should have 2 versions of GoLand 2024.app") + require.True(t, goland2024AppVersions["2024.2"], "Should have GoLand 2024.app v2024.2") + require.True(t, goland2024AppVersions["2024.3-beta"], "Should have GoLand 2024.app v2024.3-beta") + + // Verify that each host sees only their software (no renaming happens) err = ds.LoadHostSoftware(ctx, host1, false) require.NoError(t, err) - require.Len(t, host1.Software, 2, "Host1 should still have 2 software entries") + require.Len(t, host1.Software, 2, "Host1 should have 2 software entries") for _, s := range host1.Software { - require.Equal(t, "GoLand 2024.app", s.Name, "Host1 should see renamed software") + require.Equal(t, "GoLand.app", s.Name, "Host1 software should be GoLand.app") } err = ds.LoadHostSoftware(ctx, host2, false) require.NoError(t, err) require.Len(t, host2.Software, 1, "Host2 should have 1 software entry") - require.Equal(t, "GoLand 2024.app", host2.Software[0].Name, "Host2 should see renamed software") + require.Equal(t, "GoLand.app", host2.Software[0].Name, "Host2 software should be GoLand.app") + + err = ds.LoadHostSoftware(ctx, host3, false) + require.NoError(t, err) + require.Len(t, host3.Software, 2, "Host3 should have 2 software entries") + for _, s := range host3.Software { + require.Equal(t, "GoLand 2024.app", s.Name, "Host3 software should be GoLand 2024.app") + } } // Test for the bug where multiple software with the same bundle ID causes -// "software not found for checksum" errors during bundle ID rename operations. +// "software not found for checksum" errors // This test specifically validates that ALL software entries with the same -// bundle ID are properly linked to hosts when renaming occurs. +// bundle ID are properly linked to hosts func testUpdateHostSoftwareMultipleChecksumsPerBundleID(t *testing.T, ds *Datastore) { ctx := t.Context() // Note: Basic multiple versions scenario is already covered in testUpdateHostSoftwareMultipleSameBundleID - // This test focuses on the specific bug fix for renamed apps with many versions + // This test focuses on the specific bug fix for apps with many versions // First, establish the software with host1 - using 10 versions to stress test host1 := test.NewHost(t, ds, "rename-test-host1", "", "rename-key1", "rename-uuid1", time.Now()) @@ -2055,8 +2123,7 @@ func testUpdateHostSoftwareMultipleChecksumsPerBundleID(t *testing.T, ds *Datast require.NoError(t, err) require.Len(t, host1.Software, 10, "Host1 should have all 10 versions") - // Host2 reports the same software but renamed (user renamed the apps) - // This triggers the bundle ID rename logic and tests the bug fix + // Host2 reports the same software but different names host2 := test.NewHost(t, ds, "rename-test-host2", "", "rename-key2", "rename-uuid2", time.Now()) var renamedSoftware []fleet.Software @@ -2074,7 +2141,7 @@ func testUpdateHostSoftwareMultipleChecksumsPerBundleID(t *testing.T, ds *Datast require.NoError(t, err, "Should handle renamed apps with 10 versions without 'software not found for checksum' error") assert.NotNil(t, result) - // Verify the rename was processed in the database + // Verify both names exist in the database (no renaming occurs) var dbSoftware []struct { Name string `db:"name"` Version string `db:"version"` @@ -2082,31 +2149,47 @@ func testUpdateHostSoftwareMultipleChecksumsPerBundleID(t *testing.T, ds *Datast } err = ds.writer(ctx).SelectContext(ctx, &dbSoftware, `SELECT name, version, name_source FROM software - WHERE bundle_identifier = ? ORDER BY version`, + WHERE bundle_identifier = ? ORDER BY name, version`, "com.stresstest.app") require.NoError(t, err) - require.Len(t, dbSoftware, 10, "Should have 10 software entries in database") + require.Len(t, dbSoftware, 20, "Should have 20 software entries: 10 for each name") - // All should be renamed + // Verify we have 10 of each name + testAppCount := 0 + testAppRenamedCount := 0 for _, sw := range dbSoftware { - assert.Equal(t, "TestApp Renamed.app", sw.Name, "All software should use the new name") - assert.Equal(t, "bundle_4.67", sw.NameSource, "Renamed software should have bundle_4.67 source") + switch sw.Name { + case "TestApp.app": + testAppCount++ + case "TestApp Renamed.app": + testAppRenamedCount++ + } } + assert.Equal(t, 10, testAppCount, "Should have 10 'TestApp.app' entries") + assert.Equal(t, 10, testAppRenamedCount, "Should have 10 'TestApp Renamed.app' entries") - // Most importantly, verify that host2 has ALL 10 versions linked (this was the bug) + // Verify that host1 still has its original software + err = ds.LoadHostSoftware(ctx, host1, false) + require.NoError(t, err) + assert.Len(t, host1.Software, 10, "Host1 should still have all 10 versions") + for _, sw := range host1.Software { + assert.Equal(t, "TestApp.app", sw.Name, "Host1 should see original name") + } + + // Verify that host2 has ALL 10 versions linked err = ds.LoadHostSoftware(ctx, host2, false) require.NoError(t, err) - assert.Len(t, host2.Software, 10, "Host2 should have all 10 versions linked (bug fix verification)") + assert.Len(t, host2.Software, 10, "Host2 should have all 10 versions linked") - // Verify all versions are present + // Verify all versions are present for host2 versions := make(map[string]bool) for _, sw := range host2.Software { versions[sw.Version] = true - assert.Equal(t, "TestApp Renamed.app", sw.Name, "Should see renamed app") + assert.Equal(t, "TestApp Renamed.app", sw.Name, "Host2 should see its own name") } for i := 0; i < 10; i++ { version := fmt.Sprintf("1.%d.0", i) - assert.True(t, versions[version], "Should have version %s", version) + assert.True(t, versions[version], "Host2 should have version %s", version) } } @@ -9158,10 +9241,6 @@ func testPreInsertSoftwareInventory(t *testing.T, ds *Datastore) { // testUpdateHostBundleIDRenameOnlyNoNewSoftware tests if a host reports ONLY renamed software // (same bundle ID, different name) with NO new software func testUpdateHostBundleIDRenameOnlyNoNewSoftware(t *testing.T, ds *Datastore) { - // TEMPORARILY SKIPPED: updateTargetedBundleIDs is commented out for performance reasons - // TODO: Re-enable when updateTargetedBundleIDs is re-enabled - t.Skip("Skipping test: updateTargetedBundleIDs is temporarily disabled for performance reasons") - ctx := t.Context() host := test.NewHost(t, ds, "rename-test-host", "", "renamekey", "renameuuid", time.Now()) @@ -9183,7 +9262,6 @@ func testUpdateHostBundleIDRenameOnlyNoNewSoftware(t *testing.T, ds *Datastore) } // Report ONLY renamed software (same bundle IDs, different names) - // This is the edge case: NO new software, ONLY renames renamedSoftware := []fleet.Software{ {Name: "Renamed.app", Version: "1.0", Source: "apps", BundleIdentifier: "com.example.app"}, {Name: "AlsoRenamed.app", Version: "2.0", Source: "apps", BundleIdentifier: "com.example.another"}, @@ -9193,37 +9271,33 @@ func testUpdateHostBundleIDRenameOnlyNoNewSoftware(t *testing.T, ds *Datastore) _, err = ds.UpdateHostSoftware(ctx, host.ID, renamedSoftware) require.NoError(t, err) - // Verify the software entries were reused (not duplicated) + // Verify the host only has 2 pieces of sofware err = ds.LoadHostSoftware(ctx, host, false) require.NoError(t, err) require.Len(t, host.Software, 2, "Should still have exactly 2 software entries") - // Verify the IDs are the same (software was reused, not recreated) + // Verify the IDs are are not the same for _, s := range host.Software { originalID, ok := originalIDs[s.BundleIdentifier] require.True(t, ok, "Bundle ID %s should exist", s.BundleIdentifier) - require.Equal(t, originalID, s.ID, - "Software ID should be reused for bundle ID %s", s.BundleIdentifier) + require.NotEqual(t, originalID, s.ID, + "Software ID should not be reused for bundle ID %s", s.BundleIdentifier) } - // Verify no duplicate software entries were created + // Verify new software entries were created var softwareCount int err = ds.writer(ctx).GetContext(ctx, &softwareCount, `SELECT COUNT(DISTINCT id) FROM software WHERE bundle_identifier IN ('com.example.app', 'com.example.another')`) require.NoError(t, err) - require.Equal(t, 2, softwareCount, "Should have exactly 2 software entries, not duplicates") + require.Equal(t, 4, softwareCount, "Should have exactly 4 software entries") } // testUpdateHostBundleIDRenameWithNewSoftware tests the edge case where a host reports BOTH: // 1. New software that needs to be inserted -// 2. Existing software with renamed bundle IDs that needs updating +// 2. Existing software with renamed bundle IDs // This tests that both operations work correctly in the same update. func testUpdateHostBundleIDRenameWithNewSoftware(t *testing.T, ds *Datastore) { - // TEMPORARILY SKIPPED: updateTargetedBundleIDs is commented out for performance reasons - // TODO: Re-enable when updateTargetedBundleIDs is re-enabled - t.Skip("Skipping test: updateTargetedBundleIDs is temporarily disabled for performance reasons") - ctx := t.Context() host := test.NewHost(t, ds, "mixed-test-host", "", "mixedkey", "mixeduuid", time.Now()) @@ -9240,9 +9314,8 @@ func testUpdateHostBundleIDRenameWithNewSoftware(t *testing.T, ds *Datastore) { require.Len(t, host.Software, 1) slackOriginalID := host.Software[0].ID - // Step 2: Report BOTH renamed software AND new software in the same update mixedUpdate := []fleet.Software{ - // Renamed existing software (same bundle ID, different name) + // same bundle ID, different name {Name: "Slack 2.app", Version: "1.0.0", Source: "apps", BundleIdentifier: "com.tinyspeck.slackmacgap"}, // Brand new software {Name: "Chrome.app", Version: "110.0", Source: "apps", BundleIdentifier: "com.google.Chrome"}, @@ -9267,17 +9340,8 @@ func testUpdateHostBundleIDRenameWithNewSoftware(t *testing.T, ds *Datastore) { switch s.BundleIdentifier { case "com.tinyspeck.slackmacgap": foundSlack = true - // Verify Slack was renamed and ID was reused require.Equal(t, "Slack 2.app", s.Name, "Slack should be renamed") - require.Equal(t, slackOriginalID, s.ID, "Slack should reuse the same ID") - - // Verify name_source was updated - var nameSource string - err = ds.writer(ctx).GetContext(ctx, &nameSource, - `SELECT name_source FROM software WHERE id = ?`, s.ID) - require.NoError(t, err) - require.Equal(t, "bundle_4.67", nameSource, "Name source should indicate bundle ID match") - + require.NotEqual(t, slackOriginalID, s.ID) case "com.google.Chrome": foundChrome = true require.Equal(t, "Chrome.app", s.Name) @@ -9292,18 +9356,19 @@ func testUpdateHostBundleIDRenameWithNewSoftware(t *testing.T, ds *Datastore) { } } - require.True(t, foundSlack, "Should find renamed Slack") + require.True(t, foundSlack, "Should find new Slack") require.True(t, foundChrome, "Should find new Chrome") require.True(t, foundCustomTool, "Should find new CustomTool") - // Verify no duplicate software entries were created + // Verify two slack entries exist in the software table var softwareCount int err = ds.writer(ctx).GetContext(ctx, &softwareCount, `SELECT COUNT(DISTINCT id) FROM software WHERE bundle_identifier = 'com.tinyspeck.slackmacgap'`) require.NoError(t, err) - require.Equal(t, 1, softwareCount, "Should have exactly 1 Slack software entry") + require.Equal(t, 2, softwareCount, "Should have exactly 2 Slack software entries") // Verify titles were created correctly + // A new one should not have been created for Slack 2.app var titleCount int err = ds.writer(ctx).GetContext(ctx, &titleCount, `SELECT COUNT(DISTINCT id) FROM software_titles`) @@ -9506,6 +9571,30 @@ func testListHostSoftwareWithExtensionFor(t *testing.T, ds *Datastore) { require.Equal(t, "", regularApp.ExtensionFor) } +func testLongestCommonPrefix(t *testing.T, ds *Datastore) { + tests := []struct { + input []string + expected string + }{ + {input: []string{}, expected: ""}, + {input: []string{"no_common1", "another_one3"}, expected: ""}, + {input: []string{"single"}, expected: "single"}, + {input: []string{"prefix_common", "prefix_common_suffix1", "prefix_common_suffix2"}, expected: "prefix_common"}, + {input: []string{"common_prefix_suffix1", "common_prefix_suffix2", "common_prefix"}, expected: "common_prefix"}, + {input: []string{"same", "same", "same"}, expected: "same"}, + {input: []string{"partial_common1", "partial_common2", "none"}, expected: ""}, + {input: []string{"", "softwarename"}, expected: ""}, + {input: []string{"softwarename", "prefix_common", "prefix_common"}, expected: ""}, + } + + for _, tt := range tests { + t.Run(fmt.Sprintf("%v", tt.input), func(t *testing.T) { + result := longestCommonPrefix(tt.input) + require.Equal(t, tt.expected, result) + }) + } +} + // Helper function to find software by name and extension_for func findSoftware(sw []*fleet.HostSoftwareWithInstaller, name, extensionFor string) *fleet.HostSoftwareWithInstaller { for _, s := range sw { diff --git a/server/fleet/software.go b/server/fleet/software.go index 1c3ab35e5284..d13228a1d5fd 100644 --- a/server/fleet/software.go +++ b/server/fleet/software.go @@ -149,15 +149,12 @@ func (s Software) ToUniqueStr() string { // The calculation must match the one in softwareChecksumComputedColumn func (s Software) ComputeRawChecksum() ([]byte, error) { h := md5.New() //nolint:gosec // This hash is used as a DB optimization for software row lookup, not security - cols := []string{s.Version, s.Source, s.BundleIdentifier, s.Release, s.Arch, s.Vendor, s.ExtensionFor, s.ExtensionID} - // Only incorporate name if the Software is not a macOS app, because names on macOS are easily - // mutable and can lead to unintentional duplicates of Software in Fleet. - if s.Source != "apps" { - cols = append([]string{s.Name}, cols...) - } + cols := []string{s.Version, s.Source, s.BundleIdentifier, s.Release, s.Arch, s.Vendor, s.ExtensionFor, s.ExtensionID, s.Name} + if s.ApplicationID != nil && *s.ApplicationID != "" { cols = append(cols, *s.ApplicationID) } + _, err := fmt.Fprint(h, strings.Join(cols, "\x00")) if err != nil { return nil, err