feat! rewrite to parse cargo metadata instead of cargo tree output

This results in higher quality output.

Also, rewrite the debian-crate-packages script.

Sponsored-by: pep.foundation

5 files changed, 288 insertions, 113 deletions

diff --git a/Cargo.lock b/Cargo.lock
index 6aa6f1a..a043098 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -41,6 +41,37 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
 [[package]]
+name = "camino"
+version = "1.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "88ad0e1e3e88dd237a156ab9f571021b8a158caa0ae44b1968a241efb5144c1e"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "cargo-platform"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cbdb825da8a5df079a43676dbe042702f1707b1109f713a01420fbb4cc71fa27"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "cargo_metadata"
+version = "0.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3abb7553d5b9b8421c6de7cb02606ff15e0c6eea7d8eadd75ef013fd636bec36"
+dependencies = [
+ "camino",
+ "cargo-platform",
+ "semver",
+ "serde",
+ "serde_json",
+]
+
+[[package]]
 name = "cfg-if"
 version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -139,6 +170,12 @@ dependencies = [
 ]
 
 [[package]]
+name = "itoa"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6c8af84674fe1f223a982c933a0ee1086ac4d4052aa0fb8060c12c6ad838e754"
+
+[[package]]
 name = "libc"
 version = "0.2.126"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -164,6 +201,7 @@ name = "missing-dependencies"
 version = "0.1.0"
 dependencies = [
  "anyhow",
+ "cargo_metadata",
  "clap",
  "log",
  "pretty_env_logger",
@@ -259,10 +297,50 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244"
 
 [[package]]
+name = "ryu"
+version = "1.0.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09"
+
+[[package]]
 name = "semver"
 version = "1.0.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a2333e6df6d6598f2b1974829f853c2b4c5f4a6e503c10af918081aa6f8564e1"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "serde"
+version = "1.0.144"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0f747710de3dcd43b88c9168773254e809d8ddbdf9653b84e2554ab219f17860"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.144"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "94ed3a816fb1d101812f83e789f888322c34e291f894f19590dc310963e87a00"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "serde_json"
+version = "1.0.85"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e55a28e3aaef9d5ce0506d0a14dbba8054ddc7e499ef522dd8b26859ec9d4a44"
+dependencies = [
+ "itoa",
+ "ryu",
+ "serde",
+]
 
 [[package]]
 name = "strsim"
diff --git a/Cargo.toml b/Cargo.toml
index cd5f77e..fb920fb 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -7,6 +7,7 @@ edition = "2021"
 
 [dependencies]
 anyhow = "1.0.58"
+cargo_metadata = "0.15.0"
 clap = { version = "3.2.16", features = ["derive"] }
 log = "0.4.17"
 pretty_env_logger = "0.4.0"
diff --git a/README.md b/README.md
index b9df5b3..3b768b6 100644
--- a/README.md
+++ b/README.md
@@ -1,5 +1,10 @@
 # List dependencies that haven't been packaged
 
+The problems: a Rust project wants to be packaged in various operating
+systems, and want to control the Rust dependencies so that packaging
+is less work. This tool takes a list of packaged crates and versions
+and compares a Rust crate's dependencies accordingly.
+
 Operating systems such as Debian package dependencies separately, and
 frown upon embedded copies of them. Without going into why they do
 that, it's something projects using Rust have to live with, if they
@@ -7,7 +12,7 @@ want their software packaged for such systems.
 
 This repository contains tooling to make it easier to see what Rust
 dependencies (crates) are missing from a target operating system.
-There are three parts:
+There are two parts:
 
 * `debian-crate-packages` is a simple shell script that fetches the
   package list for the Debian "unstable" version, and produces a list
@@ -16,7 +21,8 @@ There are three parts:
   and produces a list of missing crates or versions.
 
 A crate may be entirely missing from the target system, or the target
-may have package of an older version.
+may have package of an older version. The Rust program looks for both
+problems.
 
 The list of missing crates can be used to make informed decisions
 about dependencies:
@@ -35,3 +41,18 @@ $ (cd /where/rust/program/is && cargo tree) | cargo run -q -- crates-in-debian
 missing-version aho-corasick 0.7.18
 missing-entirely ansi_term
 ```
+
+## FAQ
+
+### Why is only Debian supported?
+
+That's the only thing the author uses. Patches to add support for
+other operating systems would be greatly appreciated! The best way to
+do that is to write a script, similar to `debian-crate-packages`, that
+produces a list of names of crates and their versions that are
+packaged for an operating system.
+
+The format of the output is one line per crate, with name, a space,
+and the version, and nothing else.
+
+The name of the crate should be the name as used by the crate itself.
diff --git a/debian-crate-packages b/debian-crate-packages
index c4c8a5a..89f28ae 100755
--- a/debian-crate-packages
+++ b/debian-crate-packages
@@ -1,18 +1,79 @@
-#!/bin/bash
-
-set -euo pipefail
-
-if [ ! -e Packages ]; then
-	curl -s http://deb.debian.org/debian/dists/unstable/main/binary-amd64/Packages.xz |
-		unxz >Packages
-fi
-grep-dctrl -s Package,Version librust- Packages |
-	awk '
-/^Package:/ {
-  name = $2
-  gsub(/^librust-/, "", name)
-  gsub(/-dev$/, "", name)
-  gsub(/-/, "_", name)
-}
-/^Version:/ { gsub(/-[^-]*$/, "", $2); print name, $2 }
-'
+#!/usr/bin/python3
+
+import lzma
+import re
+import subprocess
+import sys
+
+
+def log(msg):
+    if True:
+        sys.stderr.write(f"{msg}\n")
+        sys.stderr.flush()
+
+
+def parse(para):
+    name = None
+    version = None
+    for line in para.splitlines():
+        if line.startswith("Package:"):
+            name = line.split()[1]
+        elif line.startswith("Version:"):
+            version = line.split()[1]
+    return name, version
+
+
+def crate_name(name):
+    if not name.startswith("librust-"):
+        return None
+    name = strip_prefix(name, "librust-")
+    name = strip_suffix(name, "-dev")
+    return name
+
+
+def strip_prefix(s, prefix):
+    if s.startswith(prefix):
+        return s[len(prefix) :]
+    else:
+        return s
+
+
+def strip_suffix(s, suffix):
+    if s.endswith(suffix):
+        return s[: -len(suffix)]
+    else:
+        return s
+
+
+def crate_version(version):
+    if "+really" in version:
+        version = version.split("really")[-1]
+        version = strip_prefix(version, ".")
+    parts = version.split("-")
+    if len(parts) == 1:
+        return parts[0]
+    else:
+        return "-".join(parts[:-1])
+
+
+output = subprocess.run(
+    [
+        "curl",
+        "-s",
+        "http://deb.debian.org/debian/dists/unstable/main/binary-amd64/Packages.xz",
+    ],
+    check=True,
+    capture_output=True,
+)
+
+packages = lzma.decompress(output.stdout).decode("UTF8")
+paras = packages.split("\n\n")
+
+for para in paras:
+    (name, version) = parse(para)
+    if name is not None and version is not None:
+        crate = crate_name(name)
+        v = crate_version(version)
+        log(f"Debian package {name} -> {crate}, {version} -> {v}")
+        if crate is not None:
+            print(crate, v)
diff --git a/src/main.rs b/src/main.rs
index 88e7f45..af31ec4 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -1,15 +1,16 @@
 use anyhow::anyhow;
+use cargo_metadata::MetadataCommand;
 use clap::Parser;
-use log::debug;
-use regex::Regex;
+use log::{debug, error, info};
 use semver::{Version, VersionReq};
+use std::collections::HashMap;
 use std::fmt;
 use std::fs::read;
 use std::path::{Path, PathBuf};
 
 fn main() {
     if let Err(e) = fallible_main() {
-        eprintln!("ERROR: {}", e);
+        error!("ERROR: {}", e);
         std::process::exit(1);
     }
 }
@@ -17,39 +18,61 @@ fn main() {
 fn fallible_main() -> anyhow::Result<()> {
     pretty_env_logger::init();
     let args = Args::parse();
-    debug!("loading packages");
-    let packages = Crates::from_file(&args.packaged)?;
-    debug!("loading dependencies");
-    let dependencies = get_dependencies()?;
-    let mut problems = Problems::default();
 
-    debug!("finding problems");
-    for c in dependencies.iter() {
-        debug!(".. {}", c.name);
-        let got = packages.versions(&c.name);
-        if got.is_empty() {
-            problems.push(Problem::Crate(c.clone()));
-        } else {
-            let wanted = VersionReq::parse(&format!("{}", c.version))?;
-            if !satisfied_by(&wanted, &got) {
-                problems.push(Problem::Version(c.clone()));
+    info!("load packaged crates");
+    let packaged = Crates::from_file(&args.packaged)?;
+
+    info!("load dependencies with version requirements");
+    let metadata = MetadataCommand::new()
+        .manifest_path(&args.dirname.join("Cargo.toml"))
+        .exec()?;
+
+    let mut dependencies: HashMap<String, VersionReq> = HashMap::new();
+    for package in metadata.packages {
+        for dep in &package.dependencies {
+            let name = canonicalize_crate_name(&package.name);
+            dependencies.insert(name, dep.req.clone());
+        }
+    }
+    info!("found {} dependencies", dependencies.len());
+
+    info!("find dependencies that are not packaged");
+    let mut problems = Problems::default();
+    for (name, req) in dependencies.iter() {
+        debug!("consider {}, version {} required", name, req);
+        if let Some(c) = packaged.get(name) {
+            if !req.matches(&c.version) {
+                info!("crate {} is packaged, but version doesn't satisfy {}", c.orig_name, c.version);
+                problems.push(Problem::MissingVersion(
+                    c.orig_name.clone(),
+                    req.clone(),
+                    c.version.clone(),
+                ));
+            } else {
+                info!("crate {} is packaged and version satisfies {}", c.orig_name, c.version);
             }
+        } else {
+            info!("crate {} is not packaged at all", name);
+            problems.push(Problem::MissingCrate(name.clone(), req.clone()));
         }
     }
 
     let mut problems = problems.to_vec();
     if problems.is_empty() {
-        debug!("all good");
+        info!("all good");
         Ok(())
     } else {
         problems.sort_by_key(|p| match p {
-            Problem::Crate(c) => (c.name.clone(), c.version.clone()),
-            Problem::Version(c) => (c.name.clone(), c.version.clone()),
+            Problem::MissingCrate(name, _) => name.clone(),
+            Problem::MissingVersion(name, _, _) => name.clone(),
         });
         for p in problems.iter() {
             println!("{}", p);
         }
-        Err(anyhow!("there were {} missing dependencies", problems.len()))
+        Err(anyhow!(
+            "there were {} missing dependencies",
+            problems.len()
+        ))
     }
 }
 
@@ -63,104 +86,95 @@ fn fallible_main() -> anyhow::Result<()> {
 struct Args {
     /// List of crates and versions packaged in the target operating system.
     packaged: PathBuf,
+
+    /// Directory of crate to analyze.
+    dirname: PathBuf,
 }
 
-fn get_dependencies() -> anyhow::Result<Crates> {
-    let pat = Regex::new(r#"(?P<name>\S+) v(?P<version>\d+\.\d+\.\d+)"#)?;
-    let lines: Result<Vec<String>, std::io::Error> = std::io::stdin().lines().collect();
-    let lines = lines?;
-    let dependencies: Result<Vec<Crate>, semver::Error> = lines.iter()
-        .filter_map(|line| {
-            if let Some(caps) = pat.captures(line) {
-                let name = caps.name("name").unwrap().as_str();
-                let version = caps.name("version").unwrap().as_str();
-                Some(Crate::new(name, version))
-            } else {
-                None
-            }
-        })
-        .collect();
-    let mut crates = Crates::default();
-    for c in dependencies? {
-        crates.push(c)
+struct Crate {
+    canonical_name: String,
+    orig_name: String,
+    version: Version,
+}
+
+impl Crate {
+    fn new(name: &str, version: Version) -> Self {
+        Self {
+            canonical_name: canonicalize_crate_name(name),
+            orig_name: name.into(),
+            version,
+        }
     }
-    Ok(crates)
 }
 
-fn satisfied_by(wanted: &VersionReq, got: &[&Version]) -> bool {
-    got.iter().any(|v| wanted.matches(v))
+fn canonicalize_crate_name(name: &str) -> String {
+    name.replace('-', "_")
 }
 
 #[derive(Default)]
 struct Crates {
-    crates: Vec<Crate>,
+    crates: HashMap<String, Crate>,
 }
 
 impl Crates {
     fn from_file(filename: &Path) -> anyhow::Result<Self> {
-        let crates: Result<Vec<Crate>, semver::Error> = String::from_utf8_lossy(&read(filename)?)
-            .lines()
-            .map(|line| line.trim())
-            .filter(|line| !line.is_empty())
-            .map(|line| {
-                let mut words = line.split(' ');
-                let name = words.next().unwrap();
-                let version = words.next().unwrap();
-                Crate::new(name, version)
-            })
-            .collect();
-        let crates = crates?;
-        Ok(Crates { crates })
-    }
-
-    fn push(&mut self, c: Crate) {
-        self.crates.push(c);
-    }
-
-    fn iter(&self) -> impl Iterator<Item = &Crate> {
-        self.crates.iter()
-    }
-
-    fn versions(&self, name: &str) -> Vec<&Version> {
-        self.crates
-            .iter()
-            .filter_map(|c| {
-                if c.name == name {
-                    Some(&c.version)
+        let data = read(filename)?;
+        let text = String::from_utf8(data)?;
+        let mut crates = Crates::default();
+
+        for (lineno, line) in text.lines().enumerate() {
+            let mut words = line.split(' ');
+            if let Some(name) = words.next() {
+                if let Some(version) = words.next() {
+                    if words.next().is_none() {
+                        let version = Version::parse(version)?;
+                        let c = Crate::new(name, version);
+                        crates.crates.insert(c.canonical_name.clone(), c);
+                    } else {
+                        return Err(anyhow!(
+                            "too many words on line {}: {}",
+                            lineno,
+                            filename.display()
+                        ));
+                    }
                 } else {
-                    None
+                    return Err(anyhow!(
+                        "no version on line {}: {}",
+                        lineno,
+                        filename.display()
+                    ));
                 }
-            })
-            .collect()
-    }
-}
+            } else {
+                return Err(anyhow!(
+                    "no name on line {}: {}",
+                    lineno,
+                    filename.display()
+                ));
+            }
+        }
 
-#[derive(Debug, Clone, Eq, PartialEq)]
-struct Crate {
-    name: String,
-    version: Version,
-}
+        Ok(crates)
+    }
 
-impl Crate {
-    fn new(name: &str, version: &str) -> Result<Self, semver::Error> {
-        Ok(Self {
-            name: name.into(),
-            version: Version::parse(version)?,
-        })
+    fn get(&self, name: &str) -> Option<&Crate> {
+        let name = canonicalize_crate_name(name);
+        self.crates.get(&name)
     }
 }
 
 #[derive(Debug, Clone, Eq, PartialEq)]
 enum Problem {
-    Crate(Crate),
-    Version(Crate),
+    MissingCrate(String, VersionReq),
+    MissingVersion(String, VersionReq, Version),
 }
 
 impl fmt::Display for Problem {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         match self {
-            Self::Crate(c) => write!(f, "missing-entirely {}", c.name),
-            Self::Version(c) => write!(f, "missing-version {} {}", c.name, c.version),
+            Self::MissingCrate(name, req) => write!(f, "missing-entirely {} {}", name, req),
+            Self::MissingVersion(name, req, got) => {
+                write!(f, "missing-version {} {} got {}", name, req, got)
+            }
         }
     }
 }