From 02c78ef0cf129f95c3bd2380fd27a02f3a0e3a7e Mon Sep 17 00:00:00 2001 From: Nima Johari Date: Sun, 14 Jul 2024 18:57:58 -0700 Subject: [PATCH] Add MemoizedWebCrawler tutorial --- Examples/MemoizedWebCrawler/.gitignore | 8 + Examples/MemoizedWebCrawler/Package.resolved | 185 +++++++++++ Examples/MemoizedWebCrawler/Package.swift | 30 ++ Examples/MemoizedWebCrawler/README.md | 310 ++++++++++++++++++ .../MemoizedWebCrawler/Keys/FetchHTTP.swift | 45 +++ .../MemoizedWebCrawler/Keys/FetchTitle.swift | 40 +++ .../Sources/MemoizedWebCrawler/Utils.swift | 17 + .../Sources/MemoizedWebCrawler/main.swift | 4 + .../MemoizedWebCrawlerTests.swift | 36 ++ 9 files changed, 675 insertions(+) create mode 100644 Examples/MemoizedWebCrawler/.gitignore create mode 100644 Examples/MemoizedWebCrawler/Package.resolved create mode 100644 Examples/MemoizedWebCrawler/Package.swift create mode 100644 Examples/MemoizedWebCrawler/README.md create mode 100644 Examples/MemoizedWebCrawler/Sources/MemoizedWebCrawler/Keys/FetchHTTP.swift create mode 100644 Examples/MemoizedWebCrawler/Sources/MemoizedWebCrawler/Keys/FetchTitle.swift create mode 100644 Examples/MemoizedWebCrawler/Sources/MemoizedWebCrawler/Utils.swift create mode 100644 Examples/MemoizedWebCrawler/Sources/MemoizedWebCrawler/main.swift create mode 100644 Examples/MemoizedWebCrawler/Tests/MemoizedWebCrawlerTests/MemoizedWebCrawlerTests.swift diff --git a/Examples/MemoizedWebCrawler/.gitignore b/Examples/MemoizedWebCrawler/.gitignore new file mode 100644 index 0000000..0023a53 --- /dev/null +++ b/Examples/MemoizedWebCrawler/.gitignore @@ -0,0 +1,8 @@ +.DS_Store +/.build +/Packages +xcuserdata/ +DerivedData/ +.swiftpm/configuration/registries.json +.swiftpm/xcode/package.xcworkspace/contents.xcworkspacedata +.netrc diff --git a/Examples/MemoizedWebCrawler/Package.resolved b/Examples/MemoizedWebCrawler/Package.resolved new file mode 100644 index 0000000..36aecef --- /dev/null +++ b/Examples/MemoizedWebCrawler/Package.resolved @@ -0,0 +1,185 @@ +{ + "pins" : [ + { + "identity" : "async-http-client", + "kind" : "remoteSourceControl", + "location" : "https://github.com/swift-server/async-http-client.git", + "state" : { + "revision" : "a22083713ee90808d527d0baa290c2fb13ca3096", + "version" : "1.21.1" + } + }, + { + "identity" : "grpc-swift", + "kind" : "remoteSourceControl", + "location" : "https://github.com/grpc/grpc-swift.git", + "state" : { + "revision" : "130467153ff0acd642d2f098b69c1ac33373b24e", + "version" : "1.15.0" + } + }, + { + "identity" : "swift-algorithms", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-algorithms", + "state" : { + "revision" : "f6919dfc309e7f1b56224378b11e28bab5bccc42", + "version" : "1.2.0" + } + }, + { + "identity" : "swift-argument-parser", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-argument-parser.git", + "state" : { + "revision" : "fee6933f37fde9a5e12a1e4aeaa93fe60116ff2a", + "version" : "1.2.2" + } + }, + { + "identity" : "swift-atomics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-atomics.git", + "state" : { + "revision" : "6c89474e62719ddcc1e9614989fff2f68208fe10", + "version" : "1.1.0" + } + }, + { + "identity" : "swift-collections", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-collections.git", + "state" : { + "revision" : "937e904258d22af6e447a0b72c0bc67583ef64a2", + "version" : "1.0.4" + } + }, + { + "identity" : "swift-crypto", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-crypto.git", + "state" : { + "revision" : "33a20e650c33f6d72d822d558333f2085effa3dc", + "version" : "2.5.0" + } + }, + { + "identity" : "swift-llbuild", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-llbuild.git", + "state" : { + "revision" : "5cd4df550b31301508a77064e3dfaa5c5628780e", + "version" : "0.5.0" + } + }, + { + "identity" : "swift-llbuild2", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-llbuild2.git", + "state" : { + "revision" : "fd8964ca6ce99bd93bc568de6e95652f9ecd9b3b", + "version" : "0.17.7" + } + }, + { + "identity" : "swift-log", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-log.git", + "state" : { + "revision" : "32e8d724467f8fe623624570367e3d50c5638e46", + "version" : "1.5.2" + } + }, + { + "identity" : "swift-nio", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio.git", + "state" : { + "revision" : "702cd7c56d5d44eeba73fdf83918339b26dc855c", + "version" : "2.62.0" + } + }, + { + "identity" : "swift-nio-extras", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio-extras.git", + "state" : { + "revision" : "0e0d0aab665ff1a0659ce75ac003081f2b1c8997", + "version" : "1.19.0" + } + }, + { + "identity" : "swift-nio-http2", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio-http2.git", + "state" : { + "revision" : "6d021a48483dbb273a9be43f65234bdc9185b364", + "version" : "1.26.0" + } + }, + { + "identity" : "swift-nio-ssl", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio-ssl.git", + "state" : { + "revision" : "e866a626e105042a6a72a870c88b4c531ba05f83", + "version" : "2.24.0" + } + }, + { + "identity" : "swift-nio-transport-services", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio-transport-services.git", + "state" : { + "revision" : "38ac8221dd20674682148d6451367f89c2652980", + "version" : "1.21.0" + } + }, + { + "identity" : "swift-numerics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-numerics.git", + "state" : { + "revision" : "0a5bc04095a675662cf24757cc0640aa2204253b", + "version" : "1.0.2" + } + }, + { + "identity" : "swift-protobuf", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-protobuf.git", + "state" : { + "revision" : "0af9125c4eae12a4973fb66574c53a54962a9e1e", + "version" : "1.21.0" + } + }, + { + "identity" : "swift-system", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-system.git", + "state" : { + "revision" : "836bc4557b74fe6d2660218d56e3ce96aff76574", + "version" : "1.1.1" + } + }, + { + "identity" : "swift-tools-support-async", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-tools-support-async.git", + "state" : { + "revision" : "68873b46fba40d6a96363f5bd5ad78e34eb2c687", + "version" : "0.9.2" + } + }, + { + "identity" : "swift-tools-support-core", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-tools-support-core.git", + "state" : { + "revision" : "93784c59434dbca8e8a9e4b700d0d6d94551da6a", + "version" : "0.5.2" + } + } + ], + "version" : 2 +} diff --git a/Examples/MemoizedWebCrawler/Package.swift b/Examples/MemoizedWebCrawler/Package.swift new file mode 100644 index 0000000..91ce053 --- /dev/null +++ b/Examples/MemoizedWebCrawler/Package.swift @@ -0,0 +1,30 @@ +// swift-tools-version: 5.9 +// The swift-tools-version declares the minimum version of Swift required to build this package. + +import PackageDescription + +let package = Package( + name: "MemoizedWebCrawler", + platforms: [ + .macOS(.v13) + ], + dependencies: [ + .package(url: "https://github.com/apple/swift-llbuild2.git", .upToNextMajor(from: "0.17.7")), + .package(url: "https://github.com/swift-server/async-http-client.git", from: "1.21.1"), + ], + targets: [ + // Targets are the basic building blocks of a package, defining a module or a test suite. + // Targets can depend on other targets in this package and products from dependencies. + .executableTarget( + name: "MemoizedWebCrawler", + dependencies: [ + .product(name: "llbuild2fx", package: "swift-llbuild2"), + .product(name: "llbuild2Util", package: "swift-llbuild2"), + .product(name: "AsyncHTTPClient", package: "async-http-client") + ] + ), + .testTarget(name: "MemoizedWebCrawlerTests", dependencies: [ + .target(name: "MemoizedWebCrawler"), + ]) + ] +) diff --git a/Examples/MemoizedWebCrawler/README.md b/Examples/MemoizedWebCrawler/README.md new file mode 100644 index 0000000..7975688 --- /dev/null +++ b/Examples/MemoizedWebCrawler/README.md @@ -0,0 +1,310 @@ +# llbuild2fx tutorial + +See also `CachedKeyTests.testHTTP`. + + +## What is llbuild2fx? + +llbuild2fx is the memoization engine that heavily embraces an underlying content-addressible storage. This makes it a suitale primitive for distributed build systems. + +With llbuild2fx you can break up a complex computation into into interdependent memoizable/cachable units called “Keys”. + +A key has the following structure: + +```swift +public struct MyExampleKeyResult: Codable { + let myExampleValue: String +} + +extension MyExampleKeyResult: FXValue {} + +public struct FetchTitle: AsyncFXKey, Encodable { + public typealias ValueType = MyExampleKeyResult + + public static let versionDependencies: [FXVersioning.Type] = [ ... ] + + let someInputParameter: String + + public init(someInputParameter: String) { + self.someInputParameter = someInputParameter + } + + public func computeValue( + _ fi: FXFunctionInterface, // A mechanism to dynamically request for other keys + _ ctx: Context + ) async throws -> MyExampleKeyResult { + // ... + let results = try await fi.request(SomeOtherKey(url: url), ctx) + // ... + return MyExampleKeyResult(myExampleValue: results.blah) + } +} +``` + +## Examples + +### A web crawler + +Suppose you are a solo-entrepreneur and you want to implement a web crawler that fetches the title of a bunch of web pages. + +#### **Example 1: Fetch titles** + +You can implement a single `FetchTitle` key, but lets decouple it into two keys: `FetchHTTP` and `FetchTitle` . + + +```swift +public struct FetchTitleResult: Codable { + let pageTitle: String +} + +extension FetchTitleResult: FXValue {} + +public struct FetchTitle: AsyncFXKey, Encodable { + public typealias ValueType = FetchTitleResult + + public static let versionDependencies: [FXVersioning.Type] = [FetchHTTP.self] + + let url: String + + public init(url: String) { + self.url = url + } + + public func computeValue(_ fi: FXFunctionInterface, _ ctx: Context) async throws -> FetchTitleResult { + let str = try await fi.request(FetchHTTP(url: url), ctx).body + + let results = try RegEx(pattern: "(.*)").matchGroups(in: str) + print(results) + if let pageTitle = results.first?.first { + return FetchTitleResult(pageTitle: pageTitle) + } else { + throw StringError("unhandled scenario") + } + } +} +``` + +and + +```swift +public struct FetchHTTPResult: Codable { + let body: String +} + +extension FetchHTTPResult: FXValue {} + +public struct FetchHTTP: AsyncFXKey, Encodable { + public typealias ValueType = FetchHTTPResult + + public static let version: Int = 2 + public static let versionDependencies: [FXVersioning.Type] = [] + + let url: String + + public init(url: String) { + self.url = url + } + + public func computeValue(_ fi: FXFunctionInterface, _ ctx: Context) async throws -> FetchHTTPResult { + let client = LLBCASFSClient(ctx.db) + + let request = HTTPClientRequest(url: self.url) + let response = try await HTTPClient.shared.execute(request, timeout: .seconds(30)) + + if response.status == .ok { + let body = try await response.body.collect(upTo: 1024 * 1024) // 1 MB + let str = String(buffer: body) + return FetchHTTPResult(body: str) + } else { + throw StringError("response.status was not ok (\(response.status))") + } + throw StringError("unhandled scenario") + } +} +``` + +This is how you would write a test for it: + +```swift + +final class CachedKeyTests: XCTestCase { + func testHTTP() async throws { + let ctx = Context() + let group = LLBMakeDefaultDispatchGroup() + + // let db = LLBInMemoryCASDatabase(group: group) + let db = LLBFileBackedCASDatabase(group: group, path: AbsolutePath("/tmp/my-cas/cas")) + + let functionCache = LLBFileBackedFunctionCache(group: group, path: AbsolutePath("/tmp/my-cas/function-cache"), version: "0") + + let executor = FXLocalExecutor() + + let engine = FXBuildEngine( + group: group, + db: db, + functionCache: functionCache, + executor: executor + ) + let results = try await engine.build(key: FetchTitle(url: "http://example.com/"), ctx).get() + XCTAssertEqual(results.pageTitle, "Example Domain") + } +} +``` + +#### **🚧 Example 2: Fetch image URLS + Fetch image blobs** + +Your solo business is doing great and you want to ship a new functionality. You want to extend your crawler to extract the list of all images from the HTML. At this point you can disconnect from the internet and rely on the fact that `FetchHTTP` is already memoized. If you want to implement `FetchListOfImages` on the airplane, you have the option to do so. llbuild2fx has populated the CAS with the right objects. On top of that it has populated the function cache, which maps already-computed keys to objects in CAS. + +Here is how you would implement `FetchListOfImages` and write a test for it: + +``` +TBD +``` + +### Decoupling “what” from “how”: A top-level “Build” key + +```swift +public struct BuildResult: Codable { + let result: TypedBuildResult +} + +extension BuildResult: FXValue {} + +public enum TypedBuildResult: Codable { + case txtFile(DataID) + case tarFile(DataID) +} + +public struct Build: AsyncFXKey, Encodable { + public typealias ValueType = BuildResult + + public static let version: Int = 1 + public static let versionDependencies: [FXVersioning.Type] = [Build.self] + + let goal: String + + public func computeValue(_ fi: FXFunctionInterface, _ ctx: Context) async throws -> BuildResult { + if goal == "release.txt" { + // Fetch release.txt via HTTP + } else if goal == "release.tar" { + // ... + } else if goal == "foo.html" { + // ... + } else if goal == "bar.png" { + // ... + } else if goal.hasPrefix("http://") { + // ... + } + } +} +``` + +### 🚧 Example 3: Shake’s release.tar example + +Another illuminating example is Shake's `release.tar` example. Consider either of these scenarios: + +* suppose you are implementing a static site generator, and as part of a final step, you want to create a tar file and scp+untar it to a remote server. +* suppose you maintain an open source project and you want to release sources for the most recent version as a tar file on web. Two important observations: + * the list of files to be included in the final compressed file is listed in `release.txt` (which can itself be dynamically generated) + * a file listed in `release.txt` may not be a leaf file and needs to be “built” on the fly. + +Here’s how you would express this in Shake: + +```haskell +import Development.Shake +import Development.Shake.FilePath + +main = shakeArgs shakeOptions $ do + want ["result.tar"] -- (1) + "*.tar" %> \out -> do -- (2) + contents <- readFileLines $ out -<.> "txt" -- (3) + need contents -- (4) + cmd "tar -cf" [out] contents -- (5) +``` + +The above snippet says + +* (1) We are interested in obtaining `result.tar` +* (2) Then it defines a rule for any target that ends with `.tar` extension + * In the body of the rule, (3) it reads content of a `txt` file with the same prefix and (4) dynamically declares dependency via `need contents` + * This may trigger building artifacts that match other rules + * (5) Once the dependencies are met, it spawns `tar -cf` passing all the paths it as CLI arg + +Here’s how it looks like in llbuild2fx: + +```swift +public struct BuildReleaseTarResult { + releaseTarID: DataID +} + +extension BuildReleaseTarResult: FXValue {} + +public struct BuildReleaseTar: AsyncFXKey, Encodable { // (2) + public typealias ValueType = BuildReleaseTarResult + + public static let version: Int = 2 + public static let versionDependencies: [FXVersioning.Type] = [Build.self] + + let releaseDotTxt: DataID // (3) + + public func computeValue(_ fi: FXFunctionInterface, _ ctx: Context) async throws -> BuildReleaseTarResult { + // Use CAS APIs to read contents of release.txt + + // Request for contens of release.txt to be built via `fi.request(Build(...))` + + // In Shake you rely on the rule system or file system primitives to read/build release.txt + // In llbuild2fx you typically pass CAS ID of the source folder (or release.txt) as input argument + // If release.txt is something that another key produces it, you would use + // fi.request(..., ctx) to obtain its CAS ID. + // (e.g. you may want to use FetchHTTP to download release.txt from a remote source.) + + // (4) Dynamically declares dependency via fi.request + // This will trigger building other keys (which may already be in function cache) + let ids = releaseTxt.lines.asyncMap { line in + try await fi.request(Build(line), ctx).get() // (4) + } + + // (5) Once the dependencies are met, it spawns tar -cf passing all the paths it as CLI arg + // TBD + + } +} +``` + +And here is a test for it + +```swift +final class CachedKeyTests: XCTestCase { + func testHTTP() async throws { + // Populate the content of release.txt + // TBD. + + let results = try await engine.build(key: Build(goal: "release.tar"), ctx).get() + + // Extract the tar file and get the list of contents + XCTAssertEqual(results.pageTitle, "Example Domain") + } +} +``` + +### Example project ideas for llbuild2fx + +* Map/Reduce cache hits from build logs +* Game of Life + * We need to port Sergio’s pre-llbuild2fx GUI example to llbuild2fx +* CASLisp + +### The FXLocalExecutor abstraction + +TBD. + +## Background + +The core reason why llbuild2fx is powerful is its reliance on content-addressable storage. Think of a tree where each node has a checksum. For each labeled node, the checksum is computed by putting together the checksum of the label, and aggregating the checksum of the children. + +You get 3 main things for free when you express your computations via llbuild2fx: + +* **(Diskless, with no extra process spawns)** You can implement in-memory commands, with little reliance on the filesystem. +* **(Memoization)** Especially if your workflows require a lot of tree transformations, expressing them as CAS transformations saves you tons of slow interactions with disk, because CAS and memoization work great together. +* **(Flexible function interface)** You get a dynamic dependency graph by the virtue of defining keys that request for other keys. + * An existing system that is *really good* at dynamic dependency graphs is Shake. llbuild2fx keys are somewhat comparable to Shake actions. But Shake (as described in [the original ICFP'12 paper](https://dl.acm.org/doi/pdf/10.1145/2398856.2364538)) does not rely on CAS (i.e. it doesn’t yield itself nicely to cloud builds) diff --git a/Examples/MemoizedWebCrawler/Sources/MemoizedWebCrawler/Keys/FetchHTTP.swift b/Examples/MemoizedWebCrawler/Sources/MemoizedWebCrawler/Keys/FetchHTTP.swift new file mode 100644 index 0000000..53e8b56 --- /dev/null +++ b/Examples/MemoizedWebCrawler/Sources/MemoizedWebCrawler/Keys/FetchHTTP.swift @@ -0,0 +1,45 @@ +import Foundation +import NIOCore +import TSFCAS +import TSFFutures +import llbuild2fx +import llbuild2 +import TSCBasic +import AsyncHTTPClient +import NIOHTTP1 + +public struct FetchHTTPResult: Codable { + let body: String +} + +extension FetchHTTPResult: FXValue {} + +public struct FetchHTTP: AsyncFXKey, Encodable { + public typealias ValueType = FetchHTTPResult + + public static let version: Int = 2 + public static let versionDependencies: [FXVersioning.Type] = [] + + let url: String + + public init(url: String) { + self.url = url + } + + public func computeValue(_ fi: FXFunctionInterface, _ ctx: Context) async throws -> FetchHTTPResult { + let client = LLBCASFSClient(ctx.db) + + let request = HTTPClientRequest(url: self.url) + let response = try await HTTPClient.shared.execute(request, timeout: .seconds(30)) + + if response.status == .ok { + let body = try await response.body.collect(upTo: 1024 * 1024) // 1 MB + let str = String(buffer: body) + return FetchHTTPResult(body: str) + } else { + throw StringError("response.status was not ok (\(response.status))") + } + throw StringError("unhandled scenario") + } +} + diff --git a/Examples/MemoizedWebCrawler/Sources/MemoizedWebCrawler/Keys/FetchTitle.swift b/Examples/MemoizedWebCrawler/Sources/MemoizedWebCrawler/Keys/FetchTitle.swift new file mode 100644 index 0000000..4b1fdf6 --- /dev/null +++ b/Examples/MemoizedWebCrawler/Sources/MemoizedWebCrawler/Keys/FetchTitle.swift @@ -0,0 +1,40 @@ +import Foundation +import NIOCore +import TSFCAS +import TSFFutures +import llbuild2fx +import llbuild2 +import TSCBasic +import AsyncHTTPClient +import NIOHTTP1 + +public struct FetchTitleResult: Codable { + let pageTitle: String +} + +extension FetchTitleResult: FXValue {} + +public struct FetchTitle: AsyncFXKey, Encodable { + public typealias ValueType = FetchTitleResult + + public static let versionDependencies: [FXVersioning.Type] = [FetchHTTP.self] + + let url: String + + public init(url: String) { + self.url = url + } + + public func computeValue(_ fi: FXFunctionInterface, _ ctx: Context) async throws -> FetchTitleResult { + let str = try await fi.request(FetchHTTP(url: url), ctx).body + + let results = try RegEx(pattern: "(.*)").matchGroups(in: str) + print(results) + if let pageTitle = results.first?.first { + return FetchTitleResult(pageTitle: pageTitle) + } else { + throw StringError("unhandled scenario") + } + } +} + diff --git a/Examples/MemoizedWebCrawler/Sources/MemoizedWebCrawler/Utils.swift b/Examples/MemoizedWebCrawler/Sources/MemoizedWebCrawler/Utils.swift new file mode 100644 index 0000000..9c52a29 --- /dev/null +++ b/Examples/MemoizedWebCrawler/Sources/MemoizedWebCrawler/Utils.swift @@ -0,0 +1,17 @@ +import Foundation +import NIOCore +import TSFCAS +import TSFFutures +import llbuild2fx +import llbuild2 +import TSCBasic + +extension LLBFileBackedFunctionCache: FXFunctionCache { + public func get(key: llbuild2.LLBKey, props: llbuild2fx.FXKeyProperties, _ ctx: TSCUtility.Context) -> TSFFutures.LLBFuture { + return get(key: key, ctx) + } + + public func update(key: llbuild2.LLBKey, props: llbuild2fx.FXKeyProperties, value: TSFCAS.LLBDataID, _ ctx: TSCUtility.Context) -> TSFFutures.LLBFuture { + return update(key: key, value: value, ctx) + } +} diff --git a/Examples/MemoizedWebCrawler/Sources/MemoizedWebCrawler/main.swift b/Examples/MemoizedWebCrawler/Sources/MemoizedWebCrawler/main.swift new file mode 100644 index 0000000..44e20d5 --- /dev/null +++ b/Examples/MemoizedWebCrawler/Sources/MemoizedWebCrawler/main.swift @@ -0,0 +1,4 @@ +// The Swift Programming Language +// https://docs.swift.org/swift-book + +print("Hello, world!") diff --git a/Examples/MemoizedWebCrawler/Tests/MemoizedWebCrawlerTests/MemoizedWebCrawlerTests.swift b/Examples/MemoizedWebCrawler/Tests/MemoizedWebCrawlerTests/MemoizedWebCrawlerTests.swift new file mode 100644 index 0000000..8facbca --- /dev/null +++ b/Examples/MemoizedWebCrawler/Tests/MemoizedWebCrawlerTests/MemoizedWebCrawlerTests.swift @@ -0,0 +1,36 @@ +@testable import MemoizedWebCrawler + +import Foundation +import AsyncHTTPClient +import NIOCore +import TSFCAS +import TSFFutures +import llbuild2fx +import llbuild2 +import TSCBasic +import Foundation + +import XCTest + +final class MemoizedWebCrawlerTests: XCTestCase { + func testHTTP() async throws { + let ctx = Context() + let group = LLBMakeDefaultDispatchGroup() + + // let db = LLBInMemoryCASDatabase(group: group) + let db = LLBFileBackedCASDatabase(group: group, path: AbsolutePath("/tmp/my-cas/cas")) + + let functionCache = LLBFileBackedFunctionCache(group: group, path: AbsolutePath("/tmp/my-cas/function-cache"), version: "0") + + let executor = FXLocalExecutor() + + let engine = FXBuildEngine( + group: group, + db: db, + functionCache: functionCache, + executor: executor + ) + let results = try await engine.build(key: FetchTitle(url: "http://example.com/"), ctx).get() + XCTAssertEqual(results.pageTitle, "Example Domain") + } +}