chore(lune): fetch and update READMEs for bins too

* Updates the `update_tools` dev script to include logic for using the
  GitHub API to fetch and write READMEs for bins.
* Also optionally accepts a GitHub token for authenticated requests
  using the `$GITHUB_TOKEN` environment variable.
* Also moves dev script dependencies at root manifest into
  `dev_dependencies`.

.lune/update_tools.luau

@@ -4,13 +4,35 @@ local process = require("@lune/process")
 local DateTime = require("@lune/datetime")
 
 local pathfs = require("../lune_packages/pathfs")
+local base64 = require("../lune_packages/base64")
 
 local manifestTypes = require("../toolchainlib/src/manifest")
 local types = require("../toolchainlib/src/utils/result_option_conv")
 local Option = types.Option
 type Option<T> = types.Option<T>
+local Result = types.Result
+type Result<T, E> = types.Result<T, E>
 local Github = require("../toolchainlib/src/github")
 
+type GithubContents = {
+	name: string,
+	path: string,
+	sha: string,
+	size: number,
+	url: string,
+	html_url: string,
+	git_url: string,
+	download_url: string,
+	type: "file" | "dir" | "symlink",
+	content: string,
+	encoding: "base64",
+	_links: {
+		self: string,
+		git: string,
+		html: string,
+	},
+}
+
 local function isoDateToTimestamp(isoDate: string): number
 	return DateTime.fromIsoDate(isoDate).unixTimestamp
 end
@@ -52,6 +74,7 @@ for _, binSrc in pathfs.readDir(BINS_SRC_DIR) do
 	local absPath = BINS_SRC_DIR:join(binSrc)
 	local binEntrypoint = absPath:join("init.luau")
 	local manifestPath = absPath:join("pesde.toml")
+	local readmePath = absPath:join("README.md")
 
 	-- Make sure our constructed entrypoint and manifest paths exist
 	assert(
@@ -69,7 +92,13 @@ for _, binSrc in pathfs.readDir(BINS_SRC_DIR) do
 	-- Make sure we have a repo name and version
 	assert(repoName ~= nil, `Failed to get repo name for tool {binSrc}`)
 
-	local gh = Github.new(repoName :: string, Option.None :: Option<Github.Config>)
+	local gh = Github.new(
+		repoName :: string,
+		Option.Some({
+			authToken = Option.from(process.env.GITHUB_TOKEN) :: Option<string>,
+			retries = Option.None :: Option<number>,
+		} :: Github.Config) :: Option<Github.Config>
+	)
 	local transactions = gh:queueTransactions({ "FetchReleases" })
 
 	-- Fetch releases, and order them by published date
@@ -125,9 +154,58 @@ for _, binSrc in pathfs.readDir(BINS_SRC_DIR) do
 
 			pathfs.writeFile(manifestPath, updatedManifest)
 		end
-
-		-- TODO: Also update the included README for the tool
 	end
+
+	-- Fetch README for the tool repo, if present
+	transactions = gh:queueTransactions({
+		{
+			type = "Custom",
+			payload = {
+				method = "GET",
+				url = `https://api.github.com/repos/{repoName}/contents/README.md`,
+			},
+		},
+	})
+
+	local contentsResp = transactions[1] :: Result<GithubContents, string>
+
+	local readmeRes = contentsResp:andThen(function(resp: GithubContents)
+		-- If there was not an error, and the contents are base64 encoded,
+		-- we decode the contents and return the decoded buffer
+		if resp.encoding == "base64" then
+			-- NOTE: Github uses a special format for encoding the contents, where it
+			-- separates the entire file into multiple lines, and encodes each line in
+			-- base64
+			local lines = string.split(resp.content, "\n")
+			local decoded = buffer.create(resp.size)
+			local cum = 0
+			for _, line in lines do
+				if line == "" then
+					-- Skip empty lines
+					continue
+				end
+
+				local src = base64.decode(buffer.fromstring(line))
+				buffer.copy(decoded, math.clamp(cum - 1, 0, math.huge), src)
+				cum += buffer.len(src)
+			end
+
+			return Result.Ok(decoded)
+		end
+
+		return Result.Err(`Unsupported encoding: {resp.encoding}`)
+	end)
+
+	-- NOTE: Ideally this block should be a match, but I cannot make use of
+	-- control flow expressions from within a function
+	if not readmeRes:isOk() then
+		warn("Failed to fetch README from tool repo:", readmeRes:unwrapErr())
+		continue
+	end
+
+	-- Write the updated README to the tool's directory
+	local readmeContents = readmeRes:unwrap()
+	pathfs.writeFile(readmePath, readmeContents)
 end
 
 local timeElapsed = string.format("%.2fs", os.clock() - START_TIME)

pesde.lock

@@ -30,8 +30,8 @@ lune = "toolchainlib"
 lune = "bins/zap"
 
 [graph."jiwonz/pathfs"."0.1.0 lune"]
-direct = ["pathfs", { name = "jiwonz/pathfs", version = "^0.1.0" }, "standard"]
-resolved_ty = "standard"
+direct = ["pathfs", { name = "jiwonz/pathfs", version = "^0.1.0" }, "dev"]
+resolved_ty = "dev"
 
 [graph."jiwonz/pathfs"."0.1.0 lune".target]
 environment = "lune"
@@ -48,8 +48,8 @@ environment = "lune"
 lib = "init.luau"
 
 [graph."lukadev_0/option"."1.2.0 lune"]
-direct = ["option", { name = "lukadev_0/option", version = "^1.2.0" }, "standard"]
-resolved_ty = "standard"
+direct = ["option", { name = "lukadev_0/option", version = "^1.2.0" }, "dev"]
+resolved_ty = "dev"
 
 [graph."lukadev_0/option"."1.2.0 lune".target]
 environment = "lune"
@@ -64,3 +64,21 @@ index_url = "https://github.com/daimond113/pesde-index"
 [graph."lukadev_0/option"."1.2.0 lune".pkg_ref.target]
 environment = "lune"
 lib = "lib/init.luau"
+
+[graph."synpixel/base64"."3.0.1 lune"]
+direct = ["base64", { name = "synpixel/base64", version = "^3.0.1" }, "standard"]
+resolved_ty = "standard"
+
+[graph."synpixel/base64"."3.0.1 lune".target]
+environment = "lune"
+lib = "lib.luau"
+
+[graph."synpixel/base64"."3.0.1 lune".pkg_ref]
+ref_ty = "pesde"
+name = "synpixel/base64"
+version = "3.0.1"
+index_url = "https://github.com/daimond113/pesde-index"
+
+[graph."synpixel/base64"."3.0.1 lune".pkg_ref.target]
+environment = "lune"
+lib = "lib.luau"

pesde.toml

@@ -7,9 +7,10 @@ workspace_members = ["toolchainlib", "bins/*"]
 [target]
 environment = "lune"
 
-[dependencies]
+[dev_dependencies]
 option = { name = "lukadev_0/option", version = "^1.2.0" }
 pathfs = { name = "jiwonz/pathfs", version = "^0.1.0" }
+base64 = { name = "synpixel/base64", version = "^3.0.1" }
 
 [indices]
 default = "https://github.com/daimond113/pesde-index"