Merge branch 'parse-metadata' into 'main'

feat! rewrite to parse cargo metadata instead of cargo tree output

See merge request sequoia-pgp/missing-dependencies!2

5 files changed, 316 insertions, 117 deletions

diff --git a/Cargo.lock b/Cargo.lock
index 6aa6f1a..a043098 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -41,6 +41,37 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
 [[package]]
+name = "camino"
+version = "1.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "88ad0e1e3e88dd237a156ab9f571021b8a158caa0ae44b1968a241efb5144c1e"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "cargo-platform"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cbdb825da8a5df079a43676dbe042702f1707b1109f713a01420fbb4cc71fa27"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "cargo_metadata"
+version = "0.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3abb7553d5b9b8421c6de7cb02606ff15e0c6eea7d8eadd75ef013fd636bec36"
+dependencies = [
+ "camino",
+ "cargo-platform",
+ "semver",
+ "serde",
+ "serde_json",
+]
+
+[[package]]
 name = "cfg-if"
 version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -139,6 +170,12 @@ dependencies = [
 ]
 
 [[package]]
+name = "itoa"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6c8af84674fe1f223a982c933a0ee1086ac4d4052aa0fb8060c12c6ad838e754"
+
+[[package]]
 name = "libc"
 version = "0.2.126"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -164,6 +201,7 @@ name = "missing-dependencies"
 version = "0.1.0"
 dependencies = [
  "anyhow",
+ "cargo_metadata",
  "clap",
  "log",
  "pretty_env_logger",
@@ -259,10 +297,50 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244"
 
 [[package]]
+name = "ryu"
+version = "1.0.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09"
+
+[[package]]
 name = "semver"
 version = "1.0.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a2333e6df6d6598f2b1974829f853c2b4c5f4a6e503c10af918081aa6f8564e1"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "serde"
+version = "1.0.144"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0f747710de3dcd43b88c9168773254e809d8ddbdf9653b84e2554ab219f17860"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.144"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "94ed3a816fb1d101812f83e789f888322c34e291f894f19590dc310963e87a00"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "serde_json"
+version = "1.0.85"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e55a28e3aaef9d5ce0506d0a14dbba8054ddc7e499ef522dd8b26859ec9d4a44"
+dependencies = [
+ "itoa",
+ "ryu",
+ "serde",
+]
 
 [[package]]
 name = "strsim"
diff --git a/Cargo.toml b/Cargo.toml
index cd5f77e..fb920fb 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -7,6 +7,7 @@ edition = "2021"
 
 [dependencies]
 anyhow = "1.0.58"
+cargo_metadata = "0.15.0"
 clap = { version = "3.2.16", features = ["derive"] }
 log = "0.4.17"
 pretty_env_logger = "0.4.0"
diff --git a/README.md b/README.md
index b9df5b3..4a0bad3 100644
--- a/README.md
+++ b/README.md
@@ -1,22 +1,36 @@
 # List dependencies that haven't been packaged
 
+The problem: a Rust project wants to be packaged in various operating
+systems, and want to control the Rust dependencies so that packaging
+is less work. Specifically, they want to know what dependencies are
+not already packaged for the target system.
+
+This tool takes a list of packaged crates and versions and compares a
+Rust crate's dependencies accordingly.
+
 Operating systems such as Debian package dependencies separately, and
 frown upon embedded copies of them. Without going into why they do
 that, it's something projects using Rust have to live with, if they
 want their software packaged for such systems.
 
+Other operating systems don't package dependencies separately. For
+those, a Rust program will just use dependencies from crates.io
+directly.
+
 This repository contains tooling to make it easier to see what Rust
 dependencies (crates) are missing from a target operating system.
-There are three parts:
+There are two parts:
 
 * `debian-crate-packages` is a simple shell script that fetches the
   package list for the Debian "unstable" version, and produces a list
   of Rust crates and versions that have been packaged.
-* The Rust program in this crate reads the outputs of the two scripts
-  and produces a list of missing crates or versions.
+
+* The Rust program in this crate reads the outputs of the script and
+  produces a list of missing crates or versions of some Rust project.
 
 A crate may be entirely missing from the target system, or the target
-may have package of an older version.
+may have package of an older version. The Rust program looks for both
+problems.
 
 The list of missing crates can be used to make informed decisions
 about dependencies:
@@ -27,11 +41,42 @@ about dependencies:
   the target system to update their package to a newer version?
 
 
-## Example
+## Example use
+
+To be run from the source tree of the `missing-dependencies` crate:
 
 ```sh
 $ ./debian-crate-packages > crates-in-debian
-$ (cd /where/rust/program/is && cargo tree) | cargo run -q -- crates-in-debian
+$ cargo run -- crates-in-debian ~/my/rust/project
 missing-version aho-corasick 0.7.18
 missing-entirely ansi_term
 ```
+
+Set `RUST_LOG` to `debug` or `info` to get log messages.
+
+
+## FAQ
+
+### Why is only Debian supported?
+
+That's the only thing the author uses. Patches to add support for
+other operating systems would be greatly appreciated! The best way to
+do that is to write a script, similar to `debian-crate-packages`, that
+produces a list of names of crates and their versions that are
+packaged for an operating system.
+
+The format of the output is one line per crate, with name, a space,
+and the version, and nothing else.
+
+The name of the crate should be the name as used by the crate itself.
+
+### Why is only Debian unstable supported?
+
+If Debian needs to package an entirely new crate, it'll be uploaded to
+unstable. Likewise new versions of existing crates, except in serious
+circumstances, mostly involving security problems. Thus, if a Rust
+project wants to consider getting packaged for Debian, it'll need to
+target unstable itself.
+
+However, if you have different use case, by all means get in touch and
+let's talk about it.
diff --git a/debian-crate-packages b/debian-crate-packages
index c4c8a5a..89f28ae 100755
--- a/debian-crate-packages
+++ b/debian-crate-packages
@@ -1,18 +1,79 @@
-#!/bin/bash
-
-set -euo pipefail
-
-if [ ! -e Packages ]; then
-	curl -s http://deb.debian.org/debian/dists/unstable/main/binary-amd64/Packages.xz |
-		unxz >Packages
-fi
-grep-dctrl -s Package,Version librust- Packages |
-	awk '
-/^Package:/ {
-  name = $2
-  gsub(/^librust-/, "", name)
-  gsub(/-dev$/, "", name)
-  gsub(/-/, "_", name)
-}
-/^Version:/ { gsub(/-[^-]*$/, "", $2); print name, $2 }
-'
+#!/usr/bin/python3
+
+import lzma
+import re
+import subprocess
+import sys
+
+
+def log(msg):
+    if True:
+        sys.stderr.write(f"{msg}\n")
+        sys.stderr.flush()
+
+
+def parse(para):
+    name = None
+    version = None
+    for line in para.splitlines():
+        if line.startswith("Package:"):
+            name = line.split()[1]
+        elif line.startswith("Version:"):
+            version = line.split()[1]
+    return name, version
+
+
+def crate_name(name):
+    if not name.startswith("librust-"):
+        return None
+    name = strip_prefix(name, "librust-")
+    name = strip_suffix(name, "-dev")
+    return name
+
+
+def strip_prefix(s, prefix):
+    if s.startswith(prefix):
+        return s[len(prefix) :]
+    else:
+        return s
+
+
+def strip_suffix(s, suffix):
+    if s.endswith(suffix):
+        return s[: -len(suffix)]
+    else:
+        return s
+
+
+def crate_version(version):
+    if "+really" in version:
+        version = version.split("really")[-1]
+        version = strip_prefix(version, ".")
+    parts = version.split("-")
+    if len(parts) == 1:
+        return parts[0]
+    else:
+        return "-".join(parts[:-1])
+
+
+output = subprocess.run(
+    [
+        "curl",
+        "-s",
+        "http://deb.debian.org/debian/dists/unstable/main/binary-amd64/Packages.xz",
+    ],
+    check=True,
+    capture_output=True,
+)
+
+packages = lzma.decompress(output.stdout).decode("UTF8")
+paras = packages.split("\n\n")
+
+for para in paras:
+    (name, version) = parse(para)
+    if name is not None and version is not None:
+        crate = crate_name(name)
+        v = crate_version(version)
+        log(f"Debian package {name} -> {crate}, {version} -> {v}")
+        if crate is not None:
+            print(crate, v)
diff --git a/src/main.rs b/src/main.rs
index 88e7f45..af31ec4 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -1,15 +1,16 @@
 use anyhow::anyhow;
+use cargo_metadata::MetadataCommand;
 use clap::Parser;
-use log::debug;
-use regex::Regex;
+use log::{debug, error, info};
 use semver::{Version, VersionReq};
+use std::collections::HashMap;
 use std::fmt;
 use std::fs::read;
 use std::path::{Path, PathBuf};
 
 fn main() {
     if let Err(e) = fallible_main() {
-        eprintln!("ERROR: {}", e);
+        error!("ERROR: {}", e);
         std::process::exit(1);
     }
 }
@@ -17,39 +18,61 @@ fn main() {
 fn fallible_main() -> anyhow::Result<()> {
     pretty_env_logger::init();
     let args = Args::parse();
-    debug!("loading packages");
-    let packages = Crates::from_file(&args.packaged)?;
-    debug!("loading dependencies");
-    let dependencies = get_dependencies()?;
-    let mut problems = Problems::default();
 
-    debug!("finding problems");
-    for c in dependencies.iter() {
-        debug!(".. {}", c.name);
-        let got = packages.versions(&c.name);
-        if got.is_empty() {
-            problems.push(Problem::Crate(c.clone()));
-        } else {
-            let wanted = VersionReq::parse(&format!("{}", c.version))?;
-            if !satisfied_by(&wanted, &got) {
-                problems.push(Problem::Version(c.clone()));
+    info!("load packaged crates");
+    let packaged = Crates::from_file(&args.packaged)?;
+
+    info!("load dependencies with version requirements");
+    let metadata = MetadataCommand::new()
+        .manifest_path(&args.dirname.join("Cargo.toml"))
+        .exec()?;
+
+    let mut dependencies: HashMap<String, VersionReq> = HashMap::new();
+    for package in metadata.packages {
+        for dep in &package.dependencies {
+            let name = canonicalize_crate_name(&package.name);
+            dependencies.insert(name, dep.req.clone());
+        }
+    }
+    info!("found {} dependencies", dependencies.len());
+
+    info!("find dependencies that are not packaged");
+    let mut problems = Problems::default();
+    for (name, req) in dependencies.iter() {
+        debug!("consider {}, version {} required", name, req);
+        if let Some(c) = packaged.get(name) {
+            if !req.matches(&c.version) {
+                info!("crate {} is packaged, but version doesn't satisfy {}", c.orig_name, c.version);
+                problems.push(Problem::MissingVersion(
+                    c.orig_name.clone(),
+                    req.clone(),
+                    c.version.clone(),
+                ));
+            } else {
+                info!("crate {} is packaged and version satisfies {}", c.orig_name, c.version);
             }
+        } else {
+            info!("crate {} is not packaged at all", name);
+            problems.push(Problem::MissingCrate(name.clone(), req.clone()));
         }
     }
 
     let mut problems = problems.to_vec();
     if problems.is_empty() {
-        debug!("all good");
+        info!("all good");
         Ok(())
     } else {
         problems.sort_by_key(|p| match p {
-            Problem::Crate(c) => (c.name.clone(), c.version.clone()),
-            Problem::Version(c) => (c.name.clone(), c.version.clone()),
+            Problem::MissingCrate(name, _) => name.clone(),
+            Problem::MissingVersion(name, _, _) => name.clone(),
         });
         for p in problems.iter() {
             println!("{}", p);
         }
-        Err(anyhow!("there were {} missing dependencies", problems.len()))
+        Err(anyhow!(
+            "there were {} missing dependencies",
+            problems.len()
+        ))
     }
 }
 
@@ -63,104 +86,95 @@ fn fallible_main() -> anyhow::Result<()> {
 struct Args {
     /// List of crates and versions packaged in the target operating system.
     packaged: PathBuf,
+
+    /// Directory of crate to analyze.
+    dirname: PathBuf,
 }
 
-fn get_dependencies() -> anyhow::Result<Crates> {
-    let pat = Regex::new(r#"(?P<name>\S+) v(?P<version>\d+\.\d+\.\d+)"#)?;
-    let lines: Result<Vec<String>, std::io::Error> = std::io::stdin().lines().collect();
-    let lines = lines?;
-    let dependencies: Result<Vec<Crate>, semver::Error> = lines.iter()
-        .filter_map(|line| {
-            if let Some(caps) = pat.captures(line) {
-                let name = caps.name("name").unwrap().as_str();
-                let version = caps.name("version").unwrap().as_str();
-                Some(Crate::new(name, version))
-            } else {
-                None
-            }
-        })
-        .collect();
-    let mut crates = Crates::default();
-    for c in dependencies? {
-        crates.push(c)
+struct Crate {
+    canonical_name: String,
+    orig_name: String,
+    version: Version,
+}
+
+impl Crate {
+    fn new(name: &str, version: Version) -> Self {
+        Self {
+            canonical_name: canonicalize_crate_name(name),
+            orig_name: name.into(),
+            version,
+        }
     }
-    Ok(crates)
 }
 
-fn satisfied_by(wanted: &VersionReq, got: &[&Version]) -> bool {
-    got.iter().any(|v| wanted.matches(v))
+fn canonicalize_crate_name(name: &str) -> String {
+    name.replace('-', "_")
 }
 
 #[derive(Default)]
 struct Crates {
-    crates: Vec<Crate>,
+    crates: HashMap<String, Crate>,
 }
 
 impl Crates {
     fn from_file(filename: &Path) -> anyhow::Result<Self> {
-        let crates: Result<Vec<Crate>, semver::Error> = String::from_utf8_lossy(&read(filename)?)
-            .lines()
-            .map(|line| line.trim())
-            .filter(|line| !line.is_empty())
-            .map(|line| {
-                let mut words = line.split(' ');
-                let name = words.next().unwrap();
-                let version = words.next().unwrap();
-                Crate::new(name, version)
-            })
-            .collect();
-        let crates = crates?;
-        Ok(Crates { crates })
-    }
-
-    fn push(&mut self, c: Crate) {
-        self.crates.push(c);
-    }
-
-    fn iter(&self) -> impl Iterator<Item = &Crate> {
-        self.crates.iter()
-    }
-
-    fn versions(&self, name: &str) -> Vec<&Version> {
-        self.crates
-            .iter()
-            .filter_map(|c| {
-                if c.name == name {
-                    Some(&c.version)
+        let data = read(filename)?;
+        let text = String::from_utf8(data)?;
+        let mut crates = Crates::default();
+
+        for (lineno, line) in text.lines().enumerate() {
+            let mut words = line.split(' ');
+            if let Some(name) = words.next() {
+                if let Some(version) = words.next() {
+                    if words.next().is_none() {
+                        let version = Version::parse(version)?;
+                        let c = Crate::new(name, version);
+                        crates.crates.insert(c.canonical_name.clone(), c);
+                    } else {
+                        return Err(anyhow!(
+                            "too many words on line {}: {}",
+                            lineno,
+                            filename.display()
+                        ));
+                    }
                 } else {
-                    None
+                    return Err(anyhow!(
+                        "no version on line {}: {}",
+                        lineno,
+                        filename.display()
+                    ));
                 }
-            })
-            .collect()
-    }
-}
+            } else {
+                return Err(anyhow!(
+                    "no name on line {}: {}",
+                    lineno,
+                    filename.display()
+                ));
+            }
+        }
 
-#[derive(Debug, Clone, Eq, PartialEq)]
-struct Crate {
-    name: String,
-    version: Version,
-}
+        Ok(crates)
+    }
 
-impl Crate {
-    fn new(name: &str, version: &str) -> Result<Self, semver::Error> {
-        Ok(Self {
-            name: name.into(),
-            version: Version::parse(version)?,
-        })
+    fn get(&self, name: &str) -> Option<&Crate> {
+        let name = canonicalize_crate_name(name);
+        self.crates.get(&name)
     }
 }
 
 #[derive(Debug, Clone, Eq, PartialEq)]
 enum Problem {
-    Crate(Crate),
-    Version(Crate),
+    MissingCrate(String, VersionReq),
+    MissingVersion(String, VersionReq, Version),
 }
 
 impl fmt::Display for Problem {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         match self {
-            Self::Crate(c) => write!(f, "missing-entirely {}", c.name),
-            Self::Version(c) => write!(f, "missing-version {} {}", c.name, c.version),
+            Self::MissingCrate(name, req) => write!(f, "missing-entirely {} {}", name, req),
+            Self::MissingVersion(name, req, got) => {
+                write!(f, "missing-version {} {} got {}", name, req, got)
+            }
         }
     }
 }