Initial Commit

.gitignore

@@ -0,0 +1,22 @@
+# This file is for zig-specific build artifacts.
+# If you have OS-specific or editor-specific files to ignore,
+# such as *.swp or .DS_Store, put those in your global
+# ~/.gitignore and put this in your ~/.gitconfig:
+#
+# [core]
+#     excludesfile = ~/.gitignore
+#
+# Cheers!
+# -andrewrk
+
+.zig-cache/
+zig-out/
+/release/
+/debug/
+/build/
+/build-*/
+/docgen_tmp/
+
+# Although this was renamed to .zig-cache, let's leave it here for a few
+# releases to make it less annoying to work with multiple branches.
+zig-cache/

.gitmodules

@@ -0,0 +1,12 @@
+[submodule "vendor/tree-sitter-zig"]
+	path = vendor/tree-sitter-zig
+	url = https://github.com/tree-sitter-grammars/tree-sitter-zig
+[submodule "vendor/tree-sitter"]
+	path = vendor/tree-sitter
+	url = https://github.com/bruvduroiu/zig-tree-sitter
+[submodule "vendor/tree-sitter-c"]
+	path = vendor/tree-sitter-c
+	url = https://github.com/tree-sitter/tree-sitter-c
+[submodule "vendor/tree-sitter-python"]
+	path = vendor/tree-sitter-python
+	url = https://github.com/tree-sitter/tree-sitter-python

README.md

@@ -0,0 +1,3 @@
+# llmstxt.zig
+
+Documentation-first approach for exposing your repository as an [llms.txt](http://llmstxt.org) file. Continouously rebuilds your `llms.txt` file to keep in sync with your codebase.

build.zig

@@ -0,0 +1,177 @@
+const std = @import("std");
+
+// Although this function looks imperative, note that its job is to
+// declaratively construct a build graph that will be executed by an external
+// runner.
+pub fn build(b: *std.Build) void {
+    // Standard target options allows the person running `zig build` to choose
+    // what target to build for. Here we do not override the defaults, which
+    // means any target is allowed, and the default is native. Other options
+    // for restricting supported target set are available.
+    const target = b.standardTargetOptions(.{});
+
+    // Standard optimization options allow the person running `zig build` to select
+    // between Debug, ReleaseSafe, ReleaseFast, and ReleaseSmall. Here we do not
+    // set a preferred release mode, allowing the user to decide how to optimize.
+    const optimize = b.standardOptimizeOption(.{});
+
+    // This creates a "module", which represents a collection of source files alongside
+    // some compilation options, such as optimization mode and linked system libraries.
+    // Every executable or library we compile will be based on one or more modules.
+    const lib_mod = b.createModule(.{
+        // `root_source_file` is the Zig "entry point" of the module. If a module
+        // only contains e.g. external object files, you can make this `null`.
+        // In this case the main source file is merely a path, however, in more
+        // complicated build scripts, this could be a generated file.
+        .root_source_file = b.path("src/root.zig"),
+        .target = target,
+        .optimize = optimize,
+    });
+
+    // We will also create a module for our other entry point, 'main.zig'.
+    const exe_mod = b.createModule(.{
+        // `root_source_file` is the Zig "entry point" of the module. If a module
+        // only contains e.g. external object files, you can make this `null`.
+        // In this case the main source file is merely a path, however, in more
+        // complicated build scripts, this could be a generated file.
+        .root_source_file = b.path("src/main.zig"),
+        .target = target,
+        .optimize = optimize,
+    });
+
+    // Modules can depend on one another using the `std.Build.Module.addImport` function.
+    // This is what allows Zig source code to use `@import("foo")` where 'foo' is not a
+    // file path. In this case, we set up `exe_mod` to import `lib_mod`.
+    exe_mod.addImport("llmstxt_zig_lib", lib_mod);
+
+    // Now, we will create a static library based on the module we created above.
+    // This creates a `std.Build.Step.Compile`, which is the build step responsible
+    // for actually invoking the compiler.
+    const lib = b.addLibrary(.{
+        .linkage = .static,
+        .name = "llmstxt_zig",
+        .root_module = lib_mod,
+    });
+
+    // This declares intent for the library to be installed into the standard
+    // location when the user invokes the "install" step (the default step when
+    // running `zig build`).
+    b.installArtifact(lib);
+
+    // This creates another `std.Build.Step.Compile`, but this one builds an executable
+    // rather than a static library.
+    const exe = b.addExecutable(.{
+        .name = "llmstxt_zig",
+        .root_module = exe_mod,
+    });
+
+    // This declares intent for the executable to be installed into the
+    // standard location when the user invokes the "install" step (the default
+    // step when running `zig build`).
+    b.installArtifact(exe);
+
+    // Tree Sitter Zig Bindings
+    const tree_sitter = b.dependency("tree_sitter", .{
+        .target = target,
+        .optimize = optimize,
+    });
+    exe.root_module.addImport("tree-sitter", tree_sitter.module("tree-sitter"));
+
+    exe.linkLibrary(tree_sitter.artifact("zig-tree-sitter"));
+
+    // Build language level packages
+    const supported_langs: [3][]const u8 = .{ "c", "python", "zig" };
+    const prefix = "tree-sitter-";
+    inline for (supported_langs) |lang| {
+        const name = comptime blk: {
+            break :blk prefix ++ lang;
+        };
+
+        const dep_root = comptime blk: {
+            break :blk "vendor/" ++ name ++ "/src/";
+        };
+
+        const langLib: *std.Build.Step.Compile = b.addStaticLibrary(.{
+            .name = name,
+            .target = target,
+            .optimize = optimize,
+            .link_libc = true,
+        });
+        langLib.addCSourceFile(.{
+            .file = b.path(comptime blk: {
+                break :blk dep_root ++ "parser.c";
+            }),
+            .flags = &.{"-std=c11"},
+        });
+
+        const scannerFilename: []const u8 = comptime blk: {
+            break :blk dep_root ++ "scanner.c";
+        };
+
+        if (hasScanner(scannerFilename)) {
+            langLib.addCSourceFile(.{
+                .file = b.path(scannerFilename),
+                .flags = &.{"-std=c11"},
+            });
+        }
+
+        langLib.addIncludePath(b.path(dep_root));
+        b.installFile(comptime blk: {
+            break :blk dep_root ++ "node-types.json";
+        }, "node-types.json");
+        b.installArtifact(langLib);
+        exe.linkLibrary(langLib);
+    }
+
+    // This *creates* a Run step in the build graph, to be executed when another
+    // step is evaluated that depends on it. The next line below will establish
+    // such a dependency.
+    const run_cmd = b.addRunArtifact(exe);
+
+    // By making the run step depend on the install step, it will be run from the
+    // installation directory rather than directly from within the cache directory.
+    // This is not necessary, however, if the application depends on other installed
+    // files, this ensures they will be present and in the expected location.
+    run_cmd.step.dependOn(b.getInstallStep());
+
+    // This allows the user to pass arguments to the application in the build
+    // command itself, like this: `zig build run -- arg1 arg2 etc`
+    if (b.args) |args| {
+        run_cmd.addArgs(args);
+    }
+
+    // This creates a build step. It will be visible in the `zig build --help` menu,
+    // and can be selected like this: `zig build run`
+    // This will evaluate the `run` step rather than the default, which is "install".
+    const run_step = b.step("run", "Run the app");
+    run_step.dependOn(&run_cmd.step);
+
+    // Creates a step for unit testing. This only builds the test executable
+    // but does not run it.
+    const lib_unit_tests = b.addTest(.{
+        .root_module = lib_mod,
+    });
+
+    const run_lib_unit_tests = b.addRunArtifact(lib_unit_tests);
+
+    const exe_unit_tests = b.addTest(.{
+        .root_module = exe_mod,
+    });
+
+    const run_exe_unit_tests = b.addRunArtifact(exe_unit_tests);
+
+    // Similar to creating the run step earlier, this exposes a `test` step to
+    // the `zig build --help` menu, providing a way for the user to request
+    // running the unit tests.
+    const test_step = b.step("test", "Run unit tests");
+    test_step.dependOn(&run_lib_unit_tests.step);
+    test_step.dependOn(&run_exe_unit_tests.step);
+}
+
+inline fn hasScanner(filename: []const u8) bool {
+    const stat = std.fs.cwd().statFile(filename) catch return false;
+    switch (stat.kind) {
+        .file => return true,
+        else => return false,
+    }
+}

build.zig.zon

@@ -0,0 +1,87 @@
+.{
+    // This is the default name used by packages depending on this one. For
+    // example, when a user runs `zig fetch --save <url>`, this field is used
+    // as the key in the `dependencies` table. Although the user can choose a
+    // different name, most users will stick with this provided value.
+    //
+    // It is redundant to include "zig" in this name because it is already
+    // within the Zig package namespace.
+    .name = .llmstxt_zig,
+
+    // This is a [Semantic Version](https://semver.org/).
+    // In a future version of Zig it will be used for package deduplication.
+    .version = "0.0.0",
+
+    // Together with name, this represents a globally unique package
+    // identifier. This field is generated by the Zig toolchain when the
+    // package is first created, and then *never changes*. This allows
+    // unambiguous detection of one package being an updated version of
+    // another.
+    //
+    // When forking a Zig project, this id should be regenerated (delete the
+    // field and run `zig build`) if the upstream project is still maintained.
+    // Otherwise, the fork is *hostile*, attempting to take control over the
+    // original project's identity. Thus it is recommended to leave the comment
+    // on the following line intact, so that it shows up in code reviews that
+    // modify the field.
+    .fingerprint = 0x2b844a91ca6e7927, // Changing this has security and trust implications.
+
+    // Tracks the earliest Zig version that the package considers to be a
+    // supported use case.
+    .minimum_zig_version = "0.14.0",
+
+    // This field is optional.
+    // Each dependency must either provide a `url` and `hash`, or a `path`.
+    // `zig build --fetch` can be used to fetch all dependencies of a package, recursively.
+    // Once all dependencies are fetched, `zig build` no longer requires
+    // internet connectivity.
+    .dependencies = .{
+        .tree_sitter = .{ .path = "./vendor/tree-sitter/" },
+        // See `zig fetch --save <url>` for a command-line interface for adding dependencies.
+        //.example = .{
+        //    // When updating this field to a new URL, be sure to delete the corresponding
+        //    // `hash`, otherwise you are communicating that you expect to find the old hash at
+        //    // the new URL. If the contents of a URL change this will result in a hash mismatch
+        //    // which will prevent zig from using it.
+        //    .url = "https://example.com/foo.tar.gz",
+        //
+        //    // This is computed from the file contents of the directory of files that is
+        //    // obtained after fetching `url` and applying the inclusion rules given by
+        //    // `paths`.
+        //    //
+        //    // This field is the source of truth; packages do not come from a `url`; they
+        //    // come from a `hash`. `url` is just one of many possible mirrors for how to
+        //    // obtain a package matching this `hash`.
+        //    //
+        //    // Uses the [multihash](https://multiformats.io/multihash/) format.
+        //    .hash = "...",
+        //
+        //    // When this is provided, the package is found in a directory relative to the
+        //    // build root. In this case the package's hash is irrelevant and therefore not
+        //    // computed. This field and `url` are mutually exclusive.
+        //    .path = "foo",
+        //
+        //    // When this is set to `true`, a package is declared to be lazily
+        //    // fetched. This makes the dependency only get fetched if it is
+        //    // actually used.
+        //    .lazy = false,
+        //},
+    },
+
+    // Specifies the set of files and directories that are included in this package.
+    // Only files and directories listed here are included in the `hash` that
+    // is computed for this package. Only files listed here will remain on disk
+    // when using the zig package manager. As a rule of thumb, one should list
+    // files required for compilation plus any license(s).
+    // Paths are relative to the build root. Use the empty string (`""`) to refer to
+    // the build root itself.
+    // A directory listed here means that all files within, recursively, are included.
+    .paths = .{
+        "build.zig",
+        "build.zig.zon",
+        "src",
+        // For example...
+        //"LICENSE",
+        //"README.md",
+    },
+}

src/main.zig

@@ -0,0 +1,36 @@
+const std = @import("std");
+const ts = @import("tree-sitter");
+
+extern fn tree_sitter_zig() callconv(.C) *ts.Language;
+
+pub fn main() !void {
+    // Create a parser for the zig language
+    const language = tree_sitter_zig();
+    defer language.destroy();
+
+    const parser = ts.Parser.create();
+    defer parser.destroy();
+    try parser.setLanguage(language);
+
+    // Parse some source code and get the root node
+    const tree = parser.parseString("pub fn main() !void {}", null);
+    defer tree.?.destroy();
+
+    const node = tree.?.rootNode();
+    std.debug.assert(std.mem.eql(u8, node.kind(), "source_file"));
+    std.debug.print("{s}", .{node.kind()});
+
+    // Create a query and execute it
+    var error_offset: u32 = 0;
+    const query = try ts.Query.create(language, "name: (identifier) @name", &error_offset);
+    defer query.destroy();
+
+    const cursor = ts.QueryCursor.create();
+    defer cursor.destroy();
+    cursor.exec(query, node);
+
+    // Get the captured node of the first match
+    const match = cursor.nextMatch().?;
+    const capture = match.captures[0].node;
+    std.debug.assert(std.mem.eql(u8, capture.kind(), "identifier"));
+}

src/root.zig

@@ -0,0 +1,13 @@
+//! By convention, root.zig is the root source file when making a library. If
+//! you are making an executable, the convention is to delete this file and
+//! start with main.zig instead.
+const std = @import("std");
+const testing = std.testing;
+
+pub export fn add(a: i32, b: i32) i32 {
+    return a + b;
+}
+
+test "basic add functionality" {
+    try testing.expect(add(3, 7) == 10);
+}

vendor/tree-sitter

@@ -0,0 +1 @@
+Subproject commit 4758423d46dab9a1b2d685c503a91fbd9647e9f1

vendor/tree-sitter-c

@@ -0,0 +1 @@
+Subproject commit 2a265d69a4caf57108a73ad2ed1e6922dd2f998c

vendor/tree-sitter-python

@@ -0,0 +1 @@
+Subproject commit 710796b8b877a970297106e5bbc8e2afa47f86ec

vendor/tree-sitter-zig

@@ -0,0 +1 @@
+Subproject commit b670c8df85a1568f498aa5c8cae42f51a90473c0