Auto merge of rust-lang#135164 - Kobzol:run-make-test-glibc-symbols, r=<try>

Add test for checking used glibc symbols

This test checks that we do not use too new glibc symbols in the compiler on x64 GNU Linux, in order not to break our [glibc promises](https://blog.rust-lang.org/2022/08/01/Increasing-glibc-kernel-requirements.html).

One thing that isn't solved in the PR yet is to make sure that this test will only run on `dist` CI, more specifically on the `dist-x86_64-linux` runner, in the opt-dist post-optimization tests (it can fail elsewhere, that doesn't matter). Any suggestions on how to do that are welcome.

Fixes: rust-lang#134037

r? `@jieyouxu`

Author: bors

src/ci/github-actions/jobs.yml

@@ -83,7 +83,7 @@ envs:
     # - not running `opt-dist`'s post-optimization smoke tests on the resulting toolchain
     #
     # If you *want* these to happen however, temporarily uncomment it before triggering a try build.
-    DIST_TRY_BUILD: 1
+#    DIST_TRY_BUILD: 1
 
   auto:
     <<: *production

src/doc/rustc-dev-guide/src/tests/directives.md

@@ -152,6 +152,8 @@ Some examples of `X` in `ignore-X` or `only-X`:
   `compare-mode-split-dwarf`, `compare-mode-split-dwarf-single`
 - The two different test modes used by coverage tests:
   `ignore-coverage-map`, `ignore-coverage-run`
+- When testing a dist toolchain: `dist`
+  - This needs to be enabled with `COMPILETEST_ENABLE_DIST_TESTS=1`
 
 The following directives will check rustc build settings and target
 settings:

src/tools/compiletest/src/directive-list.rs

@@ -175,6 +175,7 @@ const KNOWN_DIRECTIVE_NAMES: &[&str] = &[
     "only-beta",
     "only-bpf",
     "only-cdb",
+    "only-dist",
     "only-gnu",
     "only-i686-pc-windows-gnu",
     "only-i686-pc-windows-msvc",

src/tools/compiletest/src/header/cfg.rs

@@ -235,6 +235,12 @@ fn parse_cfg_name_directive<'a>(
         message: "when the test mode is {name}",
     }
 
+    condition! {
+        name: "dist",
+        condition: std::env::var("COMPILETEST_ENABLE_DIST_TESTS") == Ok("1".to_string()),
+        message: "when performing tests on dist toolchain"
+    }
+
     if prefix == "ignore" && outcome == MatchOutcome::Invalid {
         // Don't error out for ignore-tidy-* diretives, as those are not handled by compiletest.
         if name.starts_with("tidy-") {

src/tools/compiletest/src/read2.rs

@@ -38,7 +38,7 @@ pub fn read2_abbreviated(
     Ok((Output { status, stdout: stdout.into_bytes(), stderr: stderr.into_bytes() }, truncated))
 }
 
-const MAX_OUT_LEN: usize = 512 * 1024;
+const MAX_OUT_LEN: usize = 512 * 1024 * 1024;
 
 // Whenever a path is filtered when counting the length of the output, we need to add some
 // placeholder length to ensure a compiler emitting only filtered paths doesn't cause a OOM.

src/tools/opt-dist/src/tests.rs

@@ -69,6 +69,7 @@ change-id = 115898
 
 [rust]
 channel = "{channel}"
+verbose-tests = true
 
 [build]
 rustc = "{rustc}"
@@ -102,13 +103,19 @@ llvm-config = "{llvm_config}"
         "tests/incremental",
         "tests/mir-opt",
         "tests/pretty",
+        "tests/run-make/glibc-symbols-x86_64-unknown-linux-gnu",
         "tests/ui",
         "tests/crashes",
     ];
     for test_path in env.skipped_tests() {
         args.extend(["--skip", test_path]);
     }
-    cmd(&args).env("COMPILETEST_FORCE_STAGE0", "1").run().context("Cannot execute tests")
+    cmd(&args)
+        .env("COMPILETEST_FORCE_STAGE0", "1")
+        // Also run dist-only tests
+        .env("COMPILETEST_ENABLE_DIST_TESTS", "1")
+        .run()
+        .context("Cannot execute tests")
 }
 
 /// Tries to find the version of the dist artifacts (either nightly, beta, or 1.XY.Z).

tests/run-make/glibc-symbols-x86_64-unknown-linux-gnu/rmake.rs

@@ -0,0 +1,110 @@
+// Check that the compiler toolchain (rustc) that we distribute is not using newer glibc
+// symbols than a specified minimum.
+// This test should only be executed on an extracted dist archive or in a dist-* CI job.
+
+//@ only-dist
+//@ only-x86_64-unknown-linux-gnu
+//@ ignore-cross-compile
+
+use std::path::{Path, PathBuf};
+
+use run_make_support::{cmd, llvm_objdump, regex, rustc_path};
+
+fn main() {
+    // This is the maximum glibc version *supported* by the x86_64-unknown-linux-gnu target.
+    // All glibc symbols used in the compiler must be lower or equal than this version.
+    let max_supported = (2, 17, 99);
+
+    let rustc = PathBuf::from(rustc_path());
+    // Check symbols directly in rustc
+    check_symbols(&rustc, max_supported);
+
+    // Find dynamic libraries referenced by rustc that come from our lib directory
+    let lib_path = rustc.parent().unwrap().parent().unwrap().join("lib");
+    let dynamic_libs = find_dynamic_libs(&rustc)
+        .into_iter()
+        .filter_map(|path| path.canonicalize().ok())
+        .filter(|lib| lib.starts_with(&lib_path))
+        .collect::<Vec<_>>();
+    for lib in dynamic_libs {
+        check_symbols(&lib, max_supported);
+    }
+}
+
+#[derive(Debug, Ord, PartialOrd, Eq, PartialEq)]
+struct GlibcSymbol {
+    name: String,
+    version: (u32, u32, u32),
+}
+
+fn find_dynamic_libs(path: &Path) -> Vec<PathBuf> {
+    cmd("ldd")
+        .arg(path)
+        .run()
+        .stdout_utf8()
+        .lines()
+        .filter_map(|line| {
+            let line = line.trim();
+            let Some((_, line)) = line.split_once(" => ") else {
+                return None;
+            };
+            line.split_ascii_whitespace().next().map(|path| PathBuf::from(path))
+        })
+        .collect()
+}
+
+fn check_symbols(file: &Path, max_supported: (u32, u32, u32)) {
+    println!("Checking {}", file.display());
+    let mut invalid: Vec<GlibcSymbol> = get_glibc_symbols(file)
+        .into_iter()
+        .filter(|symbol| symbol.version > max_supported)
+        .collect();
+    if !invalid.is_empty() {
+        invalid.sort();
+        panic!(
+            "Found invalid glibc symbols in {}:\n{}",
+            file.display(),
+            invalid
+                .into_iter()
+                .map(|symbol| format!(
+                    "{} ({:?} higher than max allowed {:?})",
+                    symbol.name, symbol.version, max_supported
+                ))
+                .collect::<Vec<_>>()
+                .join("\n")
+        )
+    }
+}
+
+fn get_glibc_symbols(file: &Path) -> Vec<GlibcSymbol> {
+    let regex = regex::Regex::new(r#"GLIBC_(\d)+\.(\d+)(:?\.(\d+))?"#).unwrap();
+
+    // FIXME(kobzol): llvm-objdump currently chokes on the BOLTed librustc_driver.so file.
+    // Use objdump instead, since it seems to work, and we only run this test in a specific
+    // CI environment anyway.
+    cmd("objdump")
+        .arg("--dynamic-syms")
+        .arg(file)
+        .run()
+        .stdout_utf8()
+        .lines()
+        .filter_map(|line| {
+            // Example line
+            // 0000000000000000 DF *UND* 0000000000000000 (GLIBC_2.2.5) sbrk
+            let mut parts = line.split(" ").collect::<Vec<_>>().into_iter().rev();
+            let Some(name) = parts.next() else {
+                return None;
+            };
+            let Some(lib) = parts.next() else {
+                return None;
+            };
+            let Some(version) = regex.captures(lib) else {
+                return None;
+            };
+            let major = version.get(1).and_then(|m| m.as_str().parse().ok()).unwrap_or(0);
+            let minor = version.get(2).and_then(|m| m.as_str().parse().ok()).unwrap_or(0);
+            let patch = version.get(3).and_then(|m| m.as_str().parse().ok()).unwrap_or(0);
+            Some(GlibcSymbol { version: (major, minor, patch), name: name.to_string() })
+        })
+        .collect()
+}