refactor: switch to official go tree-sitter implementation #2952
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -17,14 +17,25 @@ local_path_override( | |
|
|
||
| go_deps = use_extension("@bazel_gazelle//:extensions.bzl", "go_deps") | ||
| go_deps.from_file(go_mod = "//:go.mod") | ||
| go_deps.module_override( | ||
| patches = ["//patches:go-tree-sitter-cc_library.patch"], | ||
| path = "github.com/tree-sitter/go-tree-sitter", | ||
| patch_strip = 1, | ||
| ) | ||
| go_deps.module_override( | ||
| path = "github.com/tree-sitter/tree-sitter-python", | ||
|
There was a problem hiding this comment. One alternative to patching the That's probably nicer then a patch now that I look at it again...? There was a problem hiding this comment. Using go modules works better when mixing go modules across multiple modules though, such as multiple gazelle languages from different modules. But that requires the patches :/ There was a problem hiding this comment. Making the BUILD file ourselves is basically what the dougthor42/go-tree-sitter does. Does manually setting There was a problem hiding this comment. This is only for the |
||
| patches = ["//patches:tree-sitter-python-cc_library.patch"], | ||
| patch_strip = 1, | ||
| ) | ||
| use_repo( | ||
| go_deps, | ||
| "com_github_bazelbuild_buildtools", | ||
| "com_github_bmatcuk_doublestar_v4", | ||
| "com_github_emirpasic_gods", | ||
| "com_github_ghodss_yaml", | ||
| "com_github_stretchr_testify", | ||
| "com_github_tree_sitter_go_tree_sitter", | ||
| "com_github_tree_sitter_tree_sitter_python", | ||
| "in_gopkg_yaml_v2", | ||
| "org_golang_x_sync", | ||
| ) | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -169,6 +169,13 @@ def go_deps(): | |
| sum = "h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=", | ||
| version = "v0.5.9", | ||
| ) | ||
| go_repository( | ||
| name = "com_github_mattn_go_pointer", | ||
| importpath = "github.com/mattn/go-pointer", | ||
| sum = "h1:n+XhsuGeVO6MEAp7xyEukFINEa+Quek5psIR/ylA6o0=", | ||
| version = "v0.0.1", | ||
| ) | ||
|
|
||
| go_repository( | ||
| name = "com_github_pmezard_go_difflib", | ||
| importpath = "github.com/pmezard/go-difflib", | ||
|
|
@@ -182,12 +189,7 @@ def go_deps(): | |
| sum = "h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM=", | ||
| version = "v0.0.0-20190812154241-14fe0d1b01d4", | ||
| ) | ||
|
|
||
| go_repository( | ||
| name = "com_github_stretchr_objx", | ||
| importpath = "github.com/stretchr/objx", | ||
|
|
@@ -197,8 +199,94 @@ def go_deps(): | |
| go_repository( | ||
| name = "com_github_stretchr_testify", | ||
| importpath = "github.com/stretchr/testify", | ||
| sum = "h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=", | ||
| version = "v1.10.0", | ||
| ) | ||
| go_repository( | ||
| name = "com_github_tree_sitter_go_tree_sitter", | ||
| importpath = "github.com/tree-sitter/go-tree-sitter", | ||
| sum = "h1:sx6kcg8raRFCvc9BnXglke6axya12krCJF5xJ2sftRU=", | ||
| version = "v0.25.0", | ||
| patches = [ | ||
| "//patches:go-tree-sitter-cc_library.patch", # keep | ||
| ], | ||
| patch_strip = 1, # keep | ||
| ) | ||
| go_repository( | ||
| name = "com_github_tree_sitter_tree_sitter_c", | ||
| importpath = "github.com/tree-sitter/tree-sitter-c", | ||
| sum = "h1:nBPH3FV07DzAD7p0GfNvXM+Y7pNIoPenQWBpvM++t4c=", | ||
| version = "v0.23.4", | ||
| ) | ||
| go_repository( | ||
| name = "com_github_tree_sitter_tree_sitter_cpp", | ||
| importpath = "github.com/tree-sitter/tree-sitter-cpp", | ||
| sum = "h1:LaWZsiqQKvR65yHgKmnaqA+uz6tlDJTJFCyFIeZU/8w=", | ||
| version = "v0.23.4", | ||
| ) | ||
| go_repository( | ||
| name = "com_github_tree_sitter_tree_sitter_embedded_template", | ||
| importpath = "github.com/tree-sitter/tree-sitter-embedded-template", | ||
| sum = "h1:nFkkH6Sbe56EXLmZBqHHcamTpmz3TId97I16EnGy4rg=", | ||
| version = "v0.23.2", | ||
| ) | ||
| go_repository( | ||
| name = "com_github_tree_sitter_tree_sitter_go", | ||
| importpath = "github.com/tree-sitter/tree-sitter-go", | ||
| sum = "h1:yt5KMGnTHS+86pJmLIAZMWxukr8W7Ae1STPvQUuNROA=", | ||
| version = "v0.23.4", | ||
| ) | ||
| go_repository( | ||
| name = "com_github_tree_sitter_tree_sitter_html", | ||
| importpath = "github.com/tree-sitter/tree-sitter-html", | ||
| sum = "h1:1UYDV+Yd05GGRhVnTcbP58GkKLSHHZwVaN+lBZV11Lc=", | ||
| version = "v0.23.2", | ||
| ) | ||
| go_repository( | ||
| name = "com_github_tree_sitter_tree_sitter_java", | ||
| importpath = "github.com/tree-sitter/tree-sitter-java", | ||
| sum = "h1:J9YeMGMwXYlKSP3K4Us8CitC6hjtMjqpeOf2GGo6tig=", | ||
| version = "v0.23.5", | ||
| ) | ||
| go_repository( | ||
| name = "com_github_tree_sitter_tree_sitter_javascript", | ||
| importpath = "github.com/tree-sitter/tree-sitter-javascript", | ||
| sum = "h1:1fWupaRC0ArlHJ/QJzsfQ3Ibyopw7ZfQK4xXc40Zveo=", | ||
| version = "v0.23.1", | ||
| ) | ||
| go_repository( | ||
| name = "com_github_tree_sitter_tree_sitter_json", | ||
| importpath = "github.com/tree-sitter/tree-sitter-json", | ||
| sum = "h1:tV5rMkihgtiOe14a9LHfDY5kzTl5GNUYe6carZBn0fQ=", | ||
| version = "v0.24.8", | ||
| ) | ||
| go_repository( | ||
| name = "com_github_tree_sitter_tree_sitter_php", | ||
| importpath = "github.com/tree-sitter/tree-sitter-php", | ||
| sum = "h1:iHewsLNDmznh8kgGyfWfujsZxIz1YGbSd2ZTEM0ZiP8=", | ||
| version = "v0.23.11", | ||
| ) | ||
| go_repository( | ||
| name = "com_github_tree_sitter_tree_sitter_python", | ||
| importpath = "github.com/tree-sitter/tree-sitter-python", | ||
| sum = "h1:qHnWFR5WhtMQpxBZRwiaU5Hk/29vGju6CVtmvu5Haas=", | ||
| version = "v0.23.6", | ||
| patches = [ | ||
| "//patches:tree-sitter-python-cc_library.patch", # keep | ||
| ], | ||
| patch_strip = 1, # keep | ||
| ) | ||
| go_repository( | ||
| name = "com_github_tree_sitter_tree_sitter_ruby", | ||
| importpath = "github.com/tree-sitter/tree-sitter-ruby", | ||
| sum = "h1:T/NKHUA+iVbHM440hFx+lzVOzS4dV6z8Qw8ai+72bYo=", | ||
| version = "v0.23.1", | ||
| ) | ||
| go_repository( | ||
| name = "com_github_tree_sitter_tree_sitter_rust", | ||
|
There was a problem hiding this comment. Why does this need all of the non-python languages? There was a problem hiding this comment. I don't think it needs any of them, but they are there, I think just for tests AFAICT. I don't think we should care. That's how go.mod works, and deps.bzl is how (legacy) rules_go works. |
||
| importpath = "github.com/tree-sitter/tree-sitter-rust", | ||
| sum = "h1:6AtoooCW5GqNrRpfnvl0iUhxTAZEovEmLKDbyHlfw90=", | ||
| version = "v0.23.2", | ||
| ) | ||
|
|
||
| go_repository( | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| exports_files(glob(["*.patch"])) |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,25 @@ | ||
| diff --git a/BUILD.bazel b/BUILD.bazel | ||
| index 37704d1..b6d17a8 100755 | ||
| --- a/BUILD.bazel | ||
| +++ b/BUILD.bazel | ||
| @@ -1,5 +1,12 @@ | ||
| load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") | ||
|
|
||
| +cc_library( | ||
| + name = "clib", | ||
| + hdrs = glob(["src/**/*.h", "src/**/*.c", "include/**/*.h"]), | ||
| + includes = ["include", "src"], | ||
| + visibility = ["//:__subpackages__"], | ||
| +) | ||
| + | ||
| go_library( | ||
| name = "go-tree-sitter", | ||
| srcs = [ | ||
| @@ -22,6 +30,7 @@ go_library( | ||
| "tree_sitter.go", | ||
| ], | ||
| cgo = True, | ||
| + cdeps = ["//:clib"], | ||
| copts = ["-Iinclude -Isrc -std=c11 -D_POSIX_C_SOURCE=200112L -D_DEFAULT_SOURCE"], | ||
| importpath = "github.com/tree-sitter/go-tree-sitter", | ||
| visibility = ["//visibility:public"], |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,32 @@ | ||
| diff --git a/BUILD.bazel b/BUILD.bazel | ||
| index e69de29..1f5d973 100644 | ||
| --- a/BUILD.bazel | ||
| +++ b/BUILD.bazel | ||
| @@ -0,0 +1,11 @@ | ||
| +filegroup( | ||
| + name = "csrcs", | ||
| + srcs = glob(["src/**/*.h"]), | ||
| + visibility = ["//visibility:public"], | ||
| +) | ||
| + | ||
| +cc_library( | ||
| + name = "clib", | ||
| + hdrs = ["src/parser.c", "src/scanner.c"] + glob(["src/tree_sitter/*.h"]), | ||
| + visibility = ["//bindings/go:__subpackages__"], | ||
| +) | ||
| diff --git a/bindings/go/BUILD.bazel b/bindings/go/BUILD.bazel | ||
| index f3e5232..cec7083 100644 | ||
| --- a/bindings/go/BUILD.bazel | ||
| +++ b/bindings/go/BUILD.bazel | ||
| @@ -2,9 +2,10 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") | ||
|
|
||
| go_library( | ||
| name = "go", | ||
| - srcs = ["binding.go"], | ||
| + srcs = ["binding.go", "//:csrcs"], | ||
| cgo = True, | ||
| copts = ["-std=c11 -fPIC"], | ||
| + cdeps = ["//:clib"], | ||
| importpath = "github.com/tree-sitter/tree-sitter-python/bindings/go", | ||
| visibility = ["//visibility:public"], | ||
| ) |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This is, sadly, a blocker because overrides cannot be used in non-root modules.
If this is included, then no one can add the dep via:
As it will fail with:
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
You can fork that library as well if you want, or ask users to add these 3 lines themselves. I'm just opening this PR to suggest using the modern
go-tree-sitterlibrary...There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I'm 100% for using the modern go-tree-sitter library. I just think we are blocked from doing so until bazel-contrib/rules_go#4298 is released and we update our
rules_goversion.Otherwise we get stuck in the "users must apply a patch themselves" or "we have to maintain a fork" situation that we're already in.
I think the path forward is:
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I don't think bazel-contrib/rules_go#4298 fixes every scenario. After upgrading to rules_go@HEAD I was able to remove my patch to
smacker/go-tree-sitter, but when switching totree-sitter/go-tree-sitterI had to add almost the same patch again 😢