Merge pull request #3397 from ruby/freeze-ast

Freeze AST option

Author: kddnewton

docs/ruby_api.md

@@ -23,7 +23,7 @@ The full API is documented below.
 * `Prism.parse_stream(io)` - parse the syntax tree corresponding to the source that is read out of the given IO object using the `#gets` method and return it within a parse result
 * `Prism.parse_lex(source)` - parse the syntax tree corresponding to the given source string and return it within a parse result, along with the tokens
 * `Prism.parse_lex_file(filepath)` - parse the syntax tree corresponding to the given source file and return it within a parse result, along with the tokens
-* `Prism.load(source, serialized)` - load the serialized syntax tree using the source as a reference into a syntax tree
+* `Prism.load(source, serialized, freeze = false)` - load the serialized syntax tree using the source as a reference into a syntax tree
 * `Prism.parse_comments(source)` - parse the comments corresponding to the given source string and return them
 * `Prism.parse_file_comments(source)` - parse the comments corresponding to the given source file and return them
 * `Prism.parse_success?(source)` - parse the syntax tree corresponding to the given source string and return true if it was parsed without errors

ext/prism/extension.c

@@ -7,9 +7,9 @@
 #include <ruby/encoding.h>
 #include "prism.h"
 
-VALUE pm_source_new(const pm_parser_t *parser, rb_encoding *encoding);
-VALUE pm_token_new(const pm_parser_t *parser, const pm_token_t *token, rb_encoding *encoding, VALUE source);
-VALUE pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encoding, VALUE source);
+VALUE pm_source_new(const pm_parser_t *parser, rb_encoding *encoding, bool freeze);
+VALUE pm_token_new(const pm_parser_t *parser, const pm_token_t *token, rb_encoding *encoding, VALUE source, bool freeze);
+VALUE pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encoding, VALUE source, bool freeze);
 VALUE pm_integer_new(const pm_integer_t *integer);
 
 void Init_prism_api_node(void);

ext/prism/extension.h

@@ -160,6 +160,13 @@ typedef struct pm_options {
      * inside another script.
      */
     bool partial_script;
+
+    /**
+     * Whether or not the parser should freeze the nodes that it creates. This
+     * makes it possible to have a deeply frozen AST that is safe to share
+     * between concurrency primitives.
+     */
+    bool freeze;
 } pm_options_t;
 
 /**
@@ -285,6 +292,14 @@ PRISM_EXPORTED_FUNCTION void pm_options_main_script_set(pm_options_t *options, b
  */
 PRISM_EXPORTED_FUNCTION void pm_options_partial_script_set(pm_options_t *options, bool partial_script);
 
+/**
+ * Set the freeze option on the given options struct.
+ *
+ * @param options The options struct to set the freeze value on.
+ * @param freeze The freeze value to set.
+ */
+PRISM_EXPORTED_FUNCTION void pm_options_freeze_set(pm_options_t *options, bool freeze);
+
 /**
  * Allocate and zero out the scopes array on the given options struct.
  *
@@ -355,6 +370,7 @@ PRISM_EXPORTED_FUNCTION void pm_options_free(pm_options_t *options);
  * | `1`     | encoding locked            |
  * | `1`     | main script                |
  * | `1`     | partial script             |
+ * | `1`     | freeze                     |
  * | `4`     | the number of scopes       |
  * | ...     | the scopes                 |
  *

include/prism/options.h

@@ -82,6 +82,9 @@ public static byte[] serialize(byte[] filepath, int line, byte[] encoding, boole
         // partialScript
         output.write(partialScript ? 1 : 0);
 
+        // freeze
+        output.write(0);
+
         // scopes
 
         // number of scopes

java/org/prism/ParsingOptions.java

@@ -122,6 +122,9 @@ function dumpOptions(options) {
   template.push("C");
   values.push(dumpBooleanOption(options.partial_script));
 
+  template.push("C");
+  values.push(0);
+
   template.push("L");
   if (options.scopes) {
     const scopes = options.scopes;

javascript/src/parsePrism.js

@@ -59,11 +59,11 @@ def self.lex_ripper(source)
   end
 
   # :call-seq:
-  #   Prism::load(source, serialized) -> ParseResult
+  #   Prism::load(source, serialized, freeze) -> ParseResult
   #
   # Load the serialized AST using the source as a reference into a tree.
-  def self.load(source, serialized)
-    Serialize.load(source, serialized)
+  def self.load(source, serialized, freeze = false)
+    Serialize.load_parse(source, serialized, freeze)
   end
 end
 

lib/prism.rb

@@ -15,7 +15,8 @@ module LibRubyParser # :nodoc:
     # must align with the build shared library from make/rake.
     libprism_in_build = File.expand_path("../../build/libprism.#{RbConfig::CONFIG["SOEXT"]}", __dir__)
     libprism_in_libdir = "#{RbConfig::CONFIG["libdir"]}/prism/libprism.#{RbConfig::CONFIG["SOEXT"]}"
-    if File.exist? libprism_in_build
+
+    if File.exist?(libprism_in_build)
       INCLUDE_DIR = File.expand_path("../../include", __dir__)
       ffi_lib libprism_in_build
     else
@@ -279,7 +280,7 @@ def parse_stream(stream, **options)
         # access to the IO object already through the closure of the lambda, we
         # can pass a null pointer here and not worry.
         LibRubyParser.pm_serialize_parse_stream(buffer.pointer, nil, callback, dump_options(options))
-        Prism.load(source, buffer.read)
+        Prism.load(source, buffer.read, options.fetch(:freeze, false))
       end
     end
 
@@ -354,50 +355,37 @@ def profile_file(filepath, **options)
     def dump_common(string, options) # :nodoc:
       LibRubyParser::PrismBuffer.with do |buffer|
         LibRubyParser.pm_serialize_parse(buffer.pointer, string.pointer, string.length, dump_options(options))
-        buffer.read
+
+        dumped = buffer.read
+        dumped.freeze if options.fetch(:freeze, false)
+
+        dumped
       end
     end
 
     def lex_common(string, code, options) # :nodoc:
-      serialized = LibRubyParser::PrismBuffer.with do |buffer|
+      LibRubyParser::PrismBuffer.with do |buffer|
         LibRubyParser.pm_serialize_lex(buffer.pointer, string.pointer, string.length, dump_options(options))
-        buffer.read
+        Serialize.load_lex(code, buffer.read, options.fetch(:freeze, false))
       end
-
-      Serialize.load_tokens(Source.for(code), serialized)
     end
 
     def parse_common(string, code, options) # :nodoc:
       serialized = dump_common(string, options)
-      Prism.load(code, serialized)
+      Serialize.load_parse(code, serialized, options.fetch(:freeze, false))
     end
 
     def parse_comments_common(string, code, options) # :nodoc:
       LibRubyParser::PrismBuffer.with do |buffer|
         LibRubyParser.pm_serialize_parse_comments(buffer.pointer, string.pointer, string.length, dump_options(options))
-
-        source = Source.for(code)
-        loader = Serialize::Loader.new(source, buffer.read)
-
-        loader.load_header
-        loader.load_encoding
-        loader.load_start_line
-        loader.load_comments
+        Serialize.load_parse_comments(code, buffer.read, options.fetch(:freeze, false))
       end
     end
 
     def parse_lex_common(string, code, options) # :nodoc:
       LibRubyParser::PrismBuffer.with do |buffer|
         LibRubyParser.pm_serialize_parse_lex(buffer.pointer, string.pointer, string.length, dump_options(options))
-
-        source = Source.for(code)
-        loader = Serialize::Loader.new(source, buffer.read)
-
-        tokens = loader.load_tokens
-        node, comments, magic_comments, data_loc, errors, warnings = loader.load_nodes
-        tokens.each { |token,| token.value.force_encoding(loader.encoding) }
-
-        ParseLexResult.new([node, tokens], comments, magic_comments, data_loc, errors, warnings, source)
+        Serialize.load_parse_lex(code, buffer.read, options.fetch(:freeze, false))
       end
     end
 
@@ -482,6 +470,9 @@ def dump_options(options)
       template << "C"
       values << (options.fetch(:partial_script, false) ? 1 : 0)
 
+      template << "C"
+      values << (options.fetch(:freeze, false) ? 1 : 0)
+
       template << "L"
       if (scopes = options[:scopes])
         values << scopes.length

lib/prism/ffi.rb

@@ -48,6 +48,16 @@ def initialize(source, start_line = 1, offsets = [])
       @offsets = offsets # set after parsing is done
     end
 
+    # Replace the value of start_line with the given value.
+    def replace_start_line(start_line)
+      @start_line = start_line
+    end
+
+    # Replace the value of offsets with the given value.
+    def replace_offsets(offsets)
+      @offsets.replace(offsets)
+    end
+
     # Returns the encoding of the source code, which is set by parameters to the
     # parser or by the encoding magic comment.
     def encoding
@@ -132,6 +142,13 @@ def code_units_column(byte_offset, encoding)
       code_units_offset(byte_offset, encoding) - code_units_offset(line_start(byte_offset), encoding)
     end
 
+    # Freeze this object and the objects it contains.
+    def deep_freeze
+      source.freeze
+      offsets.freeze
+      freeze
+    end
+
     private
 
     # Binary search through the offsets to find the line number for the given
@@ -854,5 +871,12 @@ def inspect
       location
       super
     end
+
+    # Freeze this object and the objects it contains.
+    def deep_freeze
+      value.freeze
+      location.freeze
+      freeze
+    end
   end
 end