Promote redex-staging to redex-stable, Redex "G" release

Summary:
allow-large-files

Sync redex-stable with the contents of redex-staging at revision 97bdf48dc1bc.

Reviewed By: ssj933

Differential Revision: D65169513

fbshipit-source-id: afe02fbb4fa407f7b4d1459c41728be60fdb242d

Author: Nikolai Tillmann

.circleci/config.yml

@@ -489,14 +489,15 @@ jobs:
   # Note: buildpack-deps:testing is the same as stable at time of writing,
   #       so jump to unstable directly.
 
-  build-deb_unstable:
-    docker:
-      - image: buildpack-deps:unstable-scm
-    # GCC is hungry.
-    resource_class: large
-    steps:
-      - build_debian:
-          job_name: debian_unstable
+  # TODO T198252082: Re-enable after migrating tests from DX to D8
+  # build-deb_unstable:
+  #   docker:
+  #     - image: buildpack-deps:unstable-scm
+  #   # GCC is hungry.
+  #   resource_class: large
+  #   steps:
+  #     - build_debian:
+  #         job_name: debian_unstable
 
   build-deb_unstable-w-clang:
     docker:

.github/actions/build_debian/action.yml renamed to .github/actions/prepare_debian/action.yml

@@ -1,4 +1,4 @@
-name: build_debian
+name: prepare_debian
 inputs:
   save_boost_cache:
     required: false
@@ -12,32 +12,35 @@ inputs:
   mode_32:
     required: false
     default: false
-  job_name:
-    required: false
 runs:
   using: composite
   steps:
+  - name: Timestamp (1)
+    run: date
+    shell: bash
   # Reinitialize. It is easier to do this (again) to not have to deal with
   # fixing `uses:` paths.
   - uses: actions/[email protected]
     with:
       path: repo
-      # We want to run restore-mtime, which needs to find actual commit data
-      # and not just the shallow default result. The easiest way for this seems
-      # to be the filter support of Github (https://github.blog/open-source/git/get-up-to-speed-with-partial-clone-and-shallow-clone/).
-      fetch-depth: 1000000
-      filter: "tree:0"
+      # We want the whole repo because of restore-mtime. Using filters is
+      # observably slower.
+      fetch-depth: 0
+  - name: Timestamp (2)
+    run: date
+    shell: bash
   - uses: ./.github/actions/debian-based-image-build-setup
     with:
       mode_32: "${{ inputs.mode_32 }}"
       install_clang: "${{ inputs.install_clang }}"
       install_clang_llvm_org: "${{ inputs.install_clang_llvm_org }}"
+  - name: Timestamp (3)
+    run: date
+    shell: bash
   - name: Run git-restore-mtime
     run: git restore-mtime
     working-directory: repo
     shell: bash
-  - uses: ./.github/actions/setup-build-and-test-w-make
-    with:
-      save_boost_cache: "${{ inputs.save_boost_cache }}"
-      mode_32: "${{ inputs.mode_32 }}"
-      job_name: "${{ inputs.job_name }}-${{ inputs.install_clang_llvm_org }}"
+  - name: Timestamp (4)
+    run: date
+    shell: bash

.github/actions/setup-build-and-test-w-make-impl/action.yml

@@ -29,12 +29,12 @@ runs:
   - name: Cache build
     uses: actions/[email protected]
     with:
-      key: v1-build-cache-${{ inputs.job_name }}.${{ inputs.setup_toolchain_extra }}.${{ inputs.configure_extra }}-${{ runner.arch }}-${{ github.ref_name }}-${{ github.sha }}
+      key: v2-build-cache-${{ inputs.job_name }}.${{ inputs.setup_toolchain_extra }}.${{ inputs.configure_extra }}-${{ runner.arch }}-${{ github.ref_name }}-${{ github.sha }}
       path: build.tar.zstd
       restore-keys: |
-        v1-build-cache-${{ inputs.job_name }}.${{ inputs.setup_toolchain_extra }}.${{ inputs.configure_extra }}-${{ runner.arch }}-${{ github.ref_name }}
-        v1-build-cache-${{ inputs.job_name }}.${{ inputs.setup_toolchain_extra }}.${{ inputs.configure_extra }}-${{ runner.arch }}-main
-        v1-build-cache-${{ inputs.job_name }}.${{ inputs.setup_toolchain_extra }}.${{ inputs.configure_extra }}-${{ runner.arch }}
+        v2-build-cache-${{ inputs.job_name }}.${{ inputs.setup_toolchain_extra }}.${{ inputs.configure_extra }}-${{ runner.arch }}-${{ github.ref_name }}-
+        v2-build-cache-${{ inputs.job_name }}.${{ inputs.setup_toolchain_extra }}.${{ inputs.configure_extra }}-${{ runner.arch }}-main-
+        v2-build-cache-${{ inputs.job_name }}.${{ inputs.setup_toolchain_extra }}.${{ inputs.configure_extra }}-${{ runner.arch }}-
   # Would be nice to use the cache action output, but we want to unpack even on
   # restore_keys hit.
   - name: Unpack build cache

.github/actions/test-build-setup/action.yml

@@ -14,9 +14,8 @@ runs:
   - name: Cache SDK
     uses: actions/[email protected]
     with:
-      key: v3-android-build-tools
+      key: v4-android-build-tools
       path: sdk.tar.zstd
-      restore-keys: v3-android-build-tools
   - name: Check/Install (SDK)
     run: |-
       if [ -e sdk/build-tools/29.0.2/dx ] ; then echo "Found SDK." ; exit 0 ; fi
@@ -41,5 +40,5 @@ runs:
       # Compress for cache. For reason see description of build cache.
       # We do not need the emulator and it is large. Delete.
       rm -rf sdk/emulator 2>/dev/null
-      tar cf - sdk | zstd -T0 -3 > sdk.tar.zstd
+      tar cf - sdk | zstd -T0 -9 > sdk.tar.zstd
     shell: bash

.github/workflows/build_main.yml

@@ -16,16 +16,23 @@ jobs:
     runs-on: 4-core-ubuntu-20.04
     steps:
     - uses: actions/[email protected]
-    - uses: "./.github/actions/build_debian"
+    - uses: "./.github/actions/prepare_debian"
+    - uses: ./.github/actions/setup-build-and-test-w-make
       with:
+        save_boost_cache: false
+        mode_32: false
         job_name: ubuntu_20.04
   build-22_04:
     runs-on: 4-core-ubuntu
     steps:
     - uses: actions/[email protected]
-    - uses: "./.github/actions/build_debian"
+    - uses: "./.github/actions/prepare_debian"
+    - uses: ./.github/actions/setup-build-and-test-w-make
       with:
+        save_boost_cache: false
+        mode_32: false
         job_name: ubuntu_22.04
+
   # No supported Github runners:
   # * debian 10
   # * debian 12
@@ -39,6 +46,12 @@ jobs:
 #      with:
 #        mode_32: true
 #        job_name: debian_12_32
+#
+# For new-style config we'd need:
+#   setup_toolchain_extra: '32'
+#   configure_extra: "--host=i686-linux-gnu CFLAGS=-m32 CXXFLAGS=-m32 LDFLAGS=-m32"
+#
+#
   build-deb_stable-w-clang-llvm-org:
     runs-on: 4-core-ubuntu
     env:
@@ -52,10 +65,15 @@ jobs:
         - 17
     steps:
     - uses: actions/[email protected]
-    - uses: "./.github/actions/build_debian"
+    - uses: "./.github/actions/prepare_debian"
       with:
         install_clang_llvm_org: "${{ matrix.clang_version }}"
-        job_name: ubuntu_22_clang_upstream
+    - uses: ./.github/actions/setup-build-and-test-w-make
+      with:
+        save_boost_cache: false
+        mode_32: false
+        job_name: "ubuntu_22_clang_upstream-${{ matrix.clang_version }}"
+
   build-windows:
     runs-on: windows-latest
     steps:
@@ -86,7 +104,11 @@ jobs:
         - 17
     steps:
     - uses: actions/[email protected]
-    - uses: "./.github/actions/build_debian"
+    - uses: "./.github/actions/prepare_debian"
       with:
         install_clang_llvm_org: "${{ matrix.clang_version }}"
-        job_name: clang_upstream
+    - uses: ./.github/actions/setup-build-and-test-w-make
+      with:
+        save_boost_cache: false
+        mode_32: false
+        job_name: "clang_upstream-${{ matrix.clang_version }}"

Makefile.am

@@ -48,7 +48,7 @@ DISABLED_WARNINGS = \
   -Wno-unused-value \
   -Wno-unused-variable
 
-AM_CXXFLAGS = --std=gnu++17 -O3 -Wall -Werror -Wextra $(DISABLED_WARNINGS) -g
+AM_CXXFLAGS = --std=gnu++17 -O3 -Wall -Werror -Wextra $(DISABLED_WARNINGS) -g1
 
 include $(top_srcdir)/Makefile.inc
 

RELEASE

@@ -1 +1 @@
-2024-08-22
+2024-10-29

libredex/ApkResources.cpp

@@ -2683,6 +2683,9 @@ bool is_inlinable_resource_value(
     android::Res_value* res_value) {
   bool is_not_complex =
       ((dtohs(entry->flags) & android::ResTable_entry::FLAG_COMPLEX) == 0);
+  if (!is_not_complex) {
+    return false;
+  }
 
   bool is_one_entry = true;
   for (auto& pair : config_to_entry_map) {
@@ -2692,17 +2695,19 @@ bool is_inlinable_resource_value(
       }
     }
   }
+  if (!is_one_entry) {
+    return false;
+  }
 
   bool is_valid_type =
       (res_value->dataType == android::Res_value::TYPE_STRING ||
        res_value->dataType == android::Res_value::TYPE_REFERENCE ||
        (res_value->dataType >= android::Res_value::TYPE_FIRST_INT &&
         res_value->dataType <= android::Res_value::TYPE_LAST_INT));
-
-  if (is_not_complex && is_one_entry && is_valid_type) {
-    return true;
+  if (!is_valid_type) {
+    return false;
   }
-  return false;
+  return true;
 }
 
 /*
@@ -2717,7 +2722,7 @@ ResourcesArscFile::get_inlinable_resource_values() {
   apk::TableEntryParser& parsed_table = table_snapshot.get_parsed_table();
   auto& res_id_to_entries = parsed_table.m_res_id_to_entries;
   std::unordered_map<uint32_t, resources::InlinableValue> inlinable_resources;
-  std::vector<std::tuple<uint32_t, android::Res_value*>> past_refs;
+  std::unordered_map<uint32_t, uint32_t> past_refs;
 
   for (auto& pair : res_id_to_entries) {
     uint32_t id = pair.first;
@@ -2748,8 +2753,7 @@ ResourcesArscFile::get_inlinable_resource_values() {
         }
         val.string_value = chars;
       } else if (res_value->dataType == android::Res_value::TYPE_REFERENCE) {
-        past_refs.push_back(
-            std::tuple<uint32_t, android::Res_value*>(id, res_value));
+        past_refs.insert({id, res_value->data});
         continue;
       } else if (res_value->dataType == android::Res_value::TYPE_INT_BOOLEAN) {
         val.bool_value = res_value->data;
@@ -2762,15 +2766,8 @@ ResourcesArscFile::get_inlinable_resource_values() {
   }
 
   // If a reference is found, check if the referenced value is inlinable and add
-  // it's actual value to the map (instead of the reference). NOTE: only works
-  // if reference only goes down one level.
-  for (auto& [id, value] : past_refs) {
-    auto it = inlinable_resources.find(value->data);
-    if (it != inlinable_resources.end()) {
-      resources::InlinableValue val = it->second;
-      inlinable_resources.insert({id, val});
-    }
-  }
+  // it's actual value to the map (instead of the reference).
+  resources::resources_inlining_find_refs(past_refs, &inlinable_resources);
   return inlinable_resources;
 }
 

libredex/BundleResources.cpp

@@ -2405,7 +2405,7 @@ std::unordered_map<uint32_t, resources::InlinableValue>
 ResourcesPbFile::get_inlinable_resource_values() {
   auto& res_id_to_configvalue = m_res_id_to_configvalue;
   std::unordered_map<uint32_t, resources::InlinableValue> inlinable_resources;
-  std::vector<std::tuple<uint32_t, uint32_t>> past_refs;
+  std::unordered_map<uint32_t, uint32_t> past_refs;
 
   for (auto& pair : res_id_to_configvalue) {
     uint32_t id = pair.first;
@@ -2434,7 +2434,7 @@ ResourcesPbFile::get_inlinable_resource_values() {
     resources::InlinableValue inlinable_val{};
     if (item.has_ref()) {
       auto id_to_add = item.ref().id();
-      past_refs.push_back(std::tuple<uint32_t, uint32_t>(id, id_to_add));
+      past_refs.insert({id, id_to_add});
       continue;
     } else if (item.has_str()) {
       const std::string& og_str = item.str().value();
@@ -2480,13 +2480,7 @@ ResourcesPbFile::get_inlinable_resource_values() {
   // If a reference is found, check if the referenced value is inlinable and add
   // it's actual value to the map (instead of the reference). NOTE: only works
   // if reference only goes down one level.
-  for (auto& [id, id_past] : past_refs) {
-    auto it = inlinable_resources.find(id_past);
-    if (it != inlinable_resources.end()) {
-      resources::InlinableValue val = it->second;
-      inlinable_resources.insert({id, val});
-    }
-  }
+  resources::resources_inlining_find_refs(past_refs, &inlinable_resources);
   return inlinable_resources;
 }
 

libredex/ControlFlow.cpp

@@ -767,9 +767,14 @@ std::ostream& operator<<(std::ostream& os, const Edge& e) {
     return os;
   }
 
-  case EDGE_THROW:
-    return os << "throw";
-
+  case EDGE_THROW: {
+    auto catch_type = e.throw_info()->catch_type;
+    if (catch_type == nullptr) {
+      return os << "throw";
+    } else {
+      return os << "throw " << show(catch_type);
+    }
+  }
   case EDGE_GHOST:
     return os << "ghost";
 

libredex/DexAccess.cpp

@@ -39,10 +39,10 @@ void loosen_access_modifier_for_vmethods(const DexClasses& scope) {
     const auto* method = pair.first;
     // If a final method has children, it can only be package-private and we can
     // not change it to be public.
-    if (is_final(method) && !pair.second->children.empty()) {
+    if (is_final(method) && !pair.second.children.empty()) {
       overriden_should_not_be_public(
-          pair.second.get(), graph.get(), &should_not_mark);
-      auto& children = pair.second->children;
+          &pair.second, graph.get(), &should_not_mark);
+      auto& children = pair.second.children;
       auto* first_child = *children.begin();
       always_assert_log(!is_public(method) && !is_protected(method),
                         "%s is visible final but it has children %s",

libredex/DexClass.cpp

@@ -641,7 +641,7 @@ std::unique_ptr<DexCode> DexCode::get_dex_code(DexIdx* idx, uint32_t offset) {
     always_assert(cdata <= end);
     always_assert((uint8_t*)(end) <= idx->end());
     while (cdata < end) {
-      DexInstruction* dop = DexInstruction::make_instruction(idx, &cdata);
+      DexInstruction* dop = DexInstruction::make_instruction(idx, &cdata, end);
       always_assert_log(dop != nullptr,
                         "Failed to parse method at offset 0x%08x", offset);
       dc->m_insns->push_back(dop);

libredex/DexInstruction.cpp

@@ -787,25 +787,32 @@ void DexInstruction::encode(DexOutputIdx* /* unused */,
 size_t DexInstruction::size() const { return m_count + 1; }
 
 DexInstruction* DexInstruction::make_instruction(DexIdx* idx,
-                                                 const uint16_t** insns_ptr) {
+                                                 const uint16_t** insns_ptr,
+                                                 const uint16_t* end) {
   auto& insns = *insns_ptr;
   auto fopcode = static_cast<DexOpcode>(*insns++);
   DexOpcode opcode = static_cast<DexOpcode>(fopcode & 0xff);
   switch (opcode) {
   case DOPCODE_NOP: {
     if (fopcode == FOPCODE_PACKED_SWITCH) {
       int count = (*insns--) * 2 + 4;
+      always_assert(count >= 0);
       insns += count;
+      always_assert(insns <= end);
       return new DexOpcodeData(insns - count, count - 1);
     } else if (fopcode == FOPCODE_SPARSE_SWITCH) {
       int count = (*insns--) * 4 + 2;
+      always_assert(count >= 0);
       insns += count;
+      always_assert(insns <= end);
       return new DexOpcodeData(insns - count, count - 1);
     } else if (fopcode == FOPCODE_FILLED_ARRAY) {
       uint16_t ewidth = *insns++;
       uint32_t size = *((uint32_t*)insns);
       int count = (ewidth * size + 1) / 2 + 4;
+      always_assert(count >= 0);
       insns += count - 2;
+      always_assert(insns <= end);
       return new DexOpcodeData(insns - count, count - 1);
     }
   }

libredex/DexInstruction.h

@@ -87,7 +87,8 @@ class DexInstruction : public Gatherable {
 
  public:
   static DexInstruction* make_instruction(DexIdx* idx,
-                                          const uint16_t** insns_ptr);
+                                          const uint16_t** insns_ptr,
+                                          const uint16_t* end);
   /* Creates the right subclass of DexInstruction for the given opcode */
   static DexInstruction* make_instruction(DexOpcode);
   virtual void encode(DexOutputIdx* dodx, uint16_t*& insns) const;

libredex/DexPosition.cpp

@@ -397,7 +397,7 @@ void RealPositionMapper::write_map_v2() {
           uint32_t parent_line = 0;
           try {
             parent_line = pos->parent == nullptr ? 0 : get_line(pos->parent);
-          } catch (std::out_of_range& e) {
+          } catch (std::out_of_range&) {
             ++unregistered_parent_positions;
             TRACE(OPUT, 1, "Parent position %s of %s was not registered",
                   SHOW(pos->parent), SHOW(pos));