Merge branch 'master' into master

Author: garbas

default.nix

@@ -140,7 +140,7 @@ public:
                     throw Error("NAR index for ‘%s’ has an unsupported version", storePath);
 
                 recurse("", ls.at("root"));
-            } catch (std::invalid_argument & e) {
+            } catch (json::parse_error & e) {
                 // FIXME: some filenames have non-UTF8 characters in them,
                 // which is not supported by nlohmann::json. So we have to
                 // skip the entire package.

file-cache.hh

@@ -0,0 +1,67 @@
+{
+  edition = 201909;
+
+  description = "Script for generating Nixpkgs/NixOS channels";
+
+  inputs.nixpkgs.url = "nixpkgs/release-19.09";
+
+  outputs = { self, nixpkgs, nix }:
+
+    {
+
+      overlay = final: prev: {
+
+        nixos-channel-scripts = with final; stdenv.mkDerivation {
+          name = "nixos-channel-scripts";
+
+          buildInputs = with final.perlPackages;
+            [ pkgconfig
+              final.nix
+              sqlite
+              makeWrapper
+              perl
+              FileSlurp
+              LWP
+              LWPProtocolHttps
+              ListMoreUtils
+              DBDSQLite
+              NetAmazonS3
+              boehmgc
+              nlohmann_json
+              boost
+            ];
+
+          buildCommand = ''
+            mkdir -p $out/bin
+
+            cp ${./file-cache.hh} file-cache.hh
+
+            g++ -Os -g ${./generate-programs-index.cc} -Wall -std=c++14 -o $out/bin/generate-programs-index -I . \
+              $(pkg-config --cflags nix-main) \
+              $(pkg-config --libs nix-main) \
+              $(pkg-config --libs nix-expr) \
+              $(pkg-config --libs nix-store) \
+              -lsqlite3 -lgc -lnixrust
+
+            g++ -Os -g ${./index-debuginfo.cc} -Wall -std=c++14 -o $out/bin/index-debuginfo -I . \
+              $(pkg-config --cflags nix-main) \
+              $(pkg-config --libs nix-main) \
+              $(pkg-config --libs nix-store) \
+              -lsqlite3 -lnixrust
+
+            cp ${./mirror-nixos-branch.pl} $out/bin/mirror-nixos-branch
+            wrapProgram $out/bin/mirror-nixos-branch --set PERL5LIB $PERL5LIB --prefix PATH : ${wget}/bin:${git}/bin:${final.nix}/bin:${gnutar}/bin:${xz}/bin:${rsync}/bin:${openssh}/bin:$out/bin
+
+            patchShebangs $out/bin
+          '';
+        };
+
+      };
+
+      defaultPackage.x86_64-linux = (import nixpkgs {
+        system = "x86_64-linux";
+        overlays = [ nix.overlay self.overlay ];
+      }).nixos-channel-scripts;
+
+    };
+}

flake.lock

@@ -49,13 +49,13 @@ void mainWrapped(int argc, char * * argv)
     auto binaryCache = openStore(binaryCacheUri).cast<BinaryCacheStore>();
 
     /* Get the allowed store paths to be included in the database. */
-    auto allowedPaths = tokenizeString<PathSet>(readFile(storePathsFile, true));
+    auto allowedPaths = binaryCache->parseStorePathSet(tokenizeString<PathSet>(readFile(storePathsFile, true)));
 
-    PathSet allowedPathsClosure;
+    StorePathSet allowedPathsClosure;
     binaryCache->computeFSClosure(allowedPaths, allowedPathsClosure);
 
-    printMsg(lvlInfo, format("%d top-level paths, %d paths in closure")
-        % allowedPaths.size() % allowedPathsClosure.size());
+    printMsg(lvlInfo, "%d top-level paths, %d paths in closure",
+        allowedPaths.size(), allowedPathsClosure.size());
 
     FileCache fileCache(cacheDbPath);
 
@@ -104,24 +104,25 @@ void mainWrapped(int argc, char * * argv)
 
     /* For each store path, figure out the package with the shortest
        attribute name. E.g. "nix" is preferred over "nixStable". */
-    std::map<Path, DrvInfo *> packagesByPath;
+    std::map<StorePath, DrvInfo *> packagesByPath;
 
     for (auto & package : packages)
         try {
             auto outputs = package.queryOutputs(true);
 
             for (auto & output : outputs) {
-                if (!allowedPathsClosure.count(output.second)) continue;
-                auto i = packagesByPath.find(output.second);
+                auto storePath = binaryCache->parseStorePath(output.second);
+                if (!allowedPathsClosure.count(storePath)) continue;
+                auto i = packagesByPath.find(storePath);
                 if (i != packagesByPath.end() &&
                     (i->second->attrPath.size() < package.attrPath.size() ||
-                     (i->second->attrPath.size() == package.attrPath.size() && i->second->attrPath < package.attrPath)))
+                        (i->second->attrPath.size() == package.attrPath.size() && i->second->attrPath < package.attrPath)))
                     continue;
-                packagesByPath[output.second] = &package;
+                packagesByPath.emplace(std::move(storePath), &package);
             }
         } catch (AssertionError & e) {
         } catch (Error & e) {
-            e.addPrefix(format("in package ‘%s’: ") % package.attrPath);
+            e.addPrefix(fmt("in package ‘%s’: ", package.attrPath));
             throw;
         }
 
@@ -203,7 +204,7 @@ void mainWrapped(int argc, char * * argv)
     ThreadPool threadPool(16);
 
     for (auto & i : packagesByPath)
-        threadPool.enqueue(std::bind(doPath, i.first, i.second));
+        threadPool.enqueue(std::bind(doPath, binaryCache->printStorePath(i.first), i.second));
 
     threadPool.process();
 

flake.nix

@@ -29,7 +29,7 @@ void mainWrapped(int argc, char * * argv)
     if (hasSuffix(binaryCacheUri, "/")) binaryCacheUri.pop_back();
     auto binaryCache = openStore(binaryCacheUri).cast<S3BinaryCacheStore>();
 
-    auto storePaths = tokenizeString<PathSet>(readFile(storePathsFile, true));
+    auto storePaths = binaryCache->parseStorePathSet(tokenizeString<PathSet>(readFile(storePathsFile, true)));
 
     std::regex debugFileRegex("^lib/debug/\\.build-id/[0-9a-f]{2}/[0-9a-f]{38}\\.debug$");
 
@@ -68,7 +68,7 @@ void mainWrapped(int argc, char * * argv)
                     std::string(file.first, prefix.size(), 2)  +
                     std::string(file.first, prefix.size() + 3, 38);
 
-                auto info = binaryCache->queryPathInfo(storePath).cast<const NarInfo>();
+                auto info = binaryCache->queryPathInfo(binaryCache->parseStorePath(storePath)).cast<const NarInfo>();
 
                 assert(hasPrefix(info->url, "nar/"));
 
@@ -84,8 +84,8 @@ void mainWrapped(int argc, char * * argv)
     };
 
     for (auto & storePath : storePaths)
-        if (hasSuffix(storePath, "-debug"))
-            threadPool.enqueue(std::bind(doPath, storePath));
+        if (hasSuffix(storePath.name(), "-debug"))
+            threadPool.enqueue(std::bind(doPath, binaryCache->printStorePath(storePath)));
 
     threadPool.process();
 }