Give an example of a properly filled infobox

- infobox requirements
- wikidata entries

Cargo.lock

@@ -360,6 +360,27 @@ dependencies = [
  "subtle",
 ]
 
+[[package]]
+name = "dirs-next"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1"
+dependencies = [
+ "cfg-if",
+ "dirs-sys-next",
+]
+
+[[package]]
+name = "dirs-sys-next"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d"
+dependencies = [
+ "libc",
+ "redox_users",
+ "winapi",
+]
+
 [[package]]
 name = "displaydoc"
 version = "0.2.5"
@@ -989,6 +1010,16 @@ version = "0.2.174"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776"
 
+[[package]]
+name = "libredox"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "391290121bad3d37fbddad76d8f5d1c1c314cfc646d143d7e07a3086ddff0ce3"
+dependencies = [
+ "bitflags 2.9.1",
+ "libc",
+]
+
 [[package]]
 name = "linux-raw-sys"
 version = "0.9.4"
@@ -1245,7 +1276,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1db05f56d34358a8b1066f67cbb203ee3e7ed2ba674a6263a1d5ec6db2204323"
 dependencies = [
  "memchr",
- "thiserror",
+ "thiserror 2.0.12",
  "ucd-trie",
 ]
 
@@ -1427,6 +1458,17 @@ dependencies = [
  "bitflags 2.9.1",
 ]
 
+[[package]]
+name = "redox_users"
+version = "0.4.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43"
+dependencies = [
+ "getrandom 0.2.16",
+ "libredox",
+ "thiserror 1.0.69",
+]
+
 [[package]]
 name = "regex"
 version = "1.11.1"
@@ -1877,13 +1919,33 @@ dependencies = [
  "unicode-width",
 ]
 
+[[package]]
+name = "thiserror"
+version = "1.0.69"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52"
+dependencies = [
+ "thiserror-impl 1.0.69",
+]
+
 [[package]]
 name = "thiserror"
 version = "2.0.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708"
 dependencies = [
- "thiserror-impl",
+ "thiserror-impl 2.0.12",
+]
+
+[[package]]
+name = "thiserror-impl"
+version = "1.0.69"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.104",
 ]
 
 [[package]]
@@ -2324,9 +2386,10 @@ dependencies = [
 
 [[package]]
 name = "wikipedia-infobox-analyzer"
-version = "0.1.0"
+version = "0.2.0"
 dependencies = [
  "ascii_table",
+ "dirs-next",
  "itertools",
  "mediawiki",
  "regex",

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "wikipedia-infobox-analyzer"
-version = "0.1.0"
+version = "0.2.0"
 edition = "2021"
 description = """
 This analysis tool allows seeing the wikidata behind articles through the lens of the infoboxes.
@@ -8,7 +8,7 @@ This analysis tool allows seeing the wikidata behind articles through the lens o
 
 authors = ["Raymon Zutekouw <contact@raymon.dev>"]
 
-repository = ""
+repository = "https://codeberg.org/MeltingSilicon/wikipedia-infobox-analyzer"
 license = "MIT OR Apache-2.0"
 
 keywords = ["wikipedia", "wikidata", "infobox"]
@@ -17,6 +17,7 @@ categories = ["command-line-utilities"]
 [dependencies]
 regex = "1.11.1"
 ascii_table = "4.0.7"
+dirs-next = "2.0.0"
 itertools = "0.14.0"
 mediawiki = "0.3.1"
 reqwest = { version = "0.12", features = ["blocking"] }

README.md

@@ -31,6 +31,55 @@ By default, the tool assumes that you are looking for tools on the English wikip
 Make sure the passed title matches the article title, and the tool should be able to find the wikidata entry.
 The next section will go over what these infobox templates files are, where you can find them on wikipedia, and how you can customize them locally for your wikidata analysis.
 
+### Interpretation of output
+The left column of the table first lists all the properties that the infobox requires.
+The right column will display the same property name if it is present in wikidata.
+If it is not present, the right column will be blank.
+Remaining properties in wikidata that are not required by the infobox are trailed in the right column.
+
+``` markdown
+┌──────────────────┬─────────────────────┬───────────────┐
+│ Infobox requires │ Wikidata Earth (Q2) │ Property name │
+├──────────────────┼─────────────────────┼───────────────┤
+│ P18              │ P18                 │ image         │
+│ P170             │ P170                │ creator       │
+│ P571             │ P571                │ inception     │
+│                  │ P31                 │               │
+│                  │ P138                │               │
+│                  │ P361                │               │
+│                  │ ....                │               │
+└──────────────────┴─────────────────────┴───────────────┘
+```
+
+This (shortened) example is complete, but if we were to try a different infobox template on the earth entry you can see that it is not a good fit.
+To demonstrate, we can apply the software template to the earth entry:
+
+``` markdown
+┌──────────────────┬─────────────────────┬───────────────┐
+│ Infobox requires │ Wikidata Earth (Q2) │ Property name │
+├──────────────────┼─────────────────────┼───────────────┤
+│ P18              │ P18                 │ image         │
+│ P154             │                     │               │
+│ P170             │ P170                │ creator       │
+│ P178             │                     │               │
+│ P275             │                     │               │
+│ P277             │                     │               │
+│ P306             │                     │               │
+│ P348             │                     │               │
+│ P400             │                     │               │
+│ P548             │                     │               │
+│ P571             │ P571                │ inception     │
+│ P577             │                     │               │
+│ P856             │                     │               │
+│ P1324            │                     │               │
+│ P2096            │                     │               │
+│                  │ P10                 │               │
+│                  │ P31                 │               │
+│                  │ P138                │               │
+│                  │ ....                │               │
+└──────────────────┴─────────────────────┴───────────────┘
+```
+
 ## Templates
 The wikipedia sites vary a lot when it comes to templates across the different languages.
 The goal of this tool is to be universal, but these templates have not been standardized as far as I am aware.
@@ -45,7 +94,7 @@ Then you can add the following line to that file locally:
 
 The program ignores what you put at the `...`.
 It permits templates that include a listing of wikidata entries for their templates.
-As an example, this is the case on the following template on the Dutch wikipedia (the first word means "uses"):
+As an example, this is the case on the following software template on the Dutch wikipedia (the first word means "uses", it can be found here: https://nl.wikipedia.org/wiki/Sjabloon:Infobox_software):
 
 ``` text
 {{Gebruikt Wikidata|P18|P154|P170|P178|P275|P277|P306|P348|P400|P548|P571|P577|P856|P1324|P2096}}
@@ -53,3 +102,36 @@ As an example, this is the case on the following template on the Dutch wikipedia
 
 Ideally, these used properties would be discovered by use, but I have not found a way to do that universally.
 Besides, you only have to do this once, and this way you can also customize the properties to look for.
+
+It is often the case that there are similar properties on a template, where to goal is that just one is filled in.
+To go back to the software example, let's take the Dutch wikipedia article for GitLab as an example.
+
+``` markdown
+┌──────────────────┬─────────────────────────────┬─────────────────────────────┐
+│ Infobox requires │ Wikidata GitLab (Q16639197) │ Property names              │
+├──────────────────┼─────────────────────────────┼─────────────────────────────┤
+│ P18              │ P18                         │ image                       │
+│ P154             │ P154                        │ logo image                  │
+│ P170             │                             │ creator                     │
+│ P178             │ P178                        │ developer                   │
+│ P275             │ P275                        │ copyright license           │
+│ P277             │ P277                        │ programmed in               │
+│ P306             │ P306                        │ operating system            │
+│ P348             │ P348                        │ software version identifier │
+│ P400             │                             │ platform                    │
+│ P548             │                             │ version type                │
+│ P571             │ P571                        │ inception                   │
+│ P577             │                             │ publication date            │
+│ P856             │ P856                        │ official website            │
+│ P1324            │ P1324                       │ source code repository URL  │
+│ P2096            │                             │ media legend                │
+│                  │ P10                         │                             │
+│                  │ P17                         │                             │
+│                  │ P31                         │                             │
+│                  │ ....                        │                             │
+└──────────────────┴─────────────────────────────┴─────────────────────────────┘
+```
+
+Here you can see that the infobox template is sufficiently filled in.
+And yes, there are some fields missing in the wikidata that the template expects, but those are covered by other properties.
+This tool should give you the ability to quickly find the missing properties to be entered in wikidata.

src/lib.rs

@@ -1,36 +1,50 @@
 #[doc = include_str!("../README.md")]
 pub mod wikipedia_infobox_analyzer {
-    use ascii_table::{Align, AsciiTable};
     use regex::Regex;
     use itertools::Itertools;
 
     /// Looks for a used template that does lists wikidata properties listing
-    pub fn extract_used_properties_from_template(template: String) -> Vec<String> {
+    pub fn extract_used_properties_from_template(template: String) -> Vec<u64> {
         let mut lines = template.lines();
         let used = lines
             .find(|line| line.starts_with("{{") && line.contains("Wikidata|"))
             .expect("Template should have a line declaring which properties are used");
 
         // The properties are listed inside the template with their number: P1|P2|P...
-        let re = Regex::new(r"P\d+").unwrap();
+        let re = Regex::new(r"P(\d+)").unwrap();
 
         // Find all matches and collect the properties into vector
-        re.find_iter(used)
-          .map(|m| m.as_str().to_owned())
+        re.captures_iter(used)
+          .map(|m| m[1].parse().unwrap())
           .collect()
     }
 
-    /// Creates an ascii infobox out of the properties
-    pub fn format_infobox_from_used_properties(name: String, properties: Vec<String>) -> String {
-        let mut ascii_table = AsciiTable::default();
-        ascii_table.set_max_width(26);
-        ascii_table
-            .column(0)
-            .set_header(name)
-            .set_align(Align::Left);
-
-        let data: Vec<Vec<&String>> = properties.iter().map(|v| vec![v]).collect();
-        ascii_table.format(data)
+    /// Properties will be displayed in the right column if they are in the list for that column
+    ///
+    /// All items in the left columns will be displayed
+    /// The remaining items in the right columns not in the left columns will trail in the right column
+    /// ```md
+    /// ┌───────┬───────┐
+    /// │   1   │   1   │
+    /// │   3   │       │
+    /// │       │   2   │
+    /// │       │   4   │
+    /// └───────┴───────┘
+    /// ```
+    /// So for example, [1, 3] and [1, 2, 4] become [1, 2, 3, 4], as shown above
+    pub fn join_columns<T: Copy + Eq>(vec1: &[T], vec2: &[T]) -> Vec<(Option<T>, Option<T>)> {
+        vec1.iter()
+            .flat_map(|n1| {
+                if let Some(&n2) = vec2.iter().find(|&x| x == n1) {
+                    std::iter::once((Some(*n1), Some(n2)))
+                } else {
+                    std::iter::once((Some(*n1), None))
+                }
+            })
+            .chain(vec2.iter()
+                   .filter(|&n2| !vec1.contains(n2))
+                   .map(|n2| (None, Some(*n2))))
+            .collect()
     }
 
     /// Fetches the wikidata label for the wiki property
@@ -86,7 +100,7 @@ pub mod wikipedia_infobox_analyzer {
            .collect::<Vec<u64>>()
     }
 
-    /// Fetch the wikidata identifier associated with the wikipedia article
+    /// Fetches the wikidata identifier associated with the wikipedia article
     pub fn fetch_wiki_item_by_article_title(title: String, language_code: String) -> u64 {
         let uri = format!("https://{}.wikipedia.org/w/api.php?action=query&prop=pageprops&format=json&titles={}", &language_code, &title);
 
@@ -104,7 +118,7 @@ pub mod wikipedia_infobox_analyzer {
 }
 
 #[cfg(test)]
-mod tests {
+mod tests_lib {
     use super::wikipedia_infobox_analyzer::*;
     use std::fs;
 
@@ -116,9 +130,9 @@ mod tests {
         assert_eq!(
             extract_used_properties_from_template(contents),
             vec![
-                "P18", "P154", "P1128", "P4103", "P2139", "P2295", "P2226", "P856", "P169",
-                "P1448", "P159", "P749", "P355", "P17", "P1056", "P452", "P576", "P112", "P127",
-                "P856", "P2096"
+                18, 154, 1128, 4103, 2139, 2295, 2226, 856, 169,
+                1448, 159, 749, 355, 17, 1056, 452, 576, 112, 127,
+                856, 2096
             ]
         );
 
@@ -128,27 +142,8 @@ mod tests {
         assert_eq!(
             extract_used_properties_from_template(contents),
             vec![
-                "P18", "P154", "P170", "P178", "P275", "P277", "P306", "P348", "P400", "P548",
-                "P571", "P577", "P856", "P1324", "P2096"
-            ]
-        );
-    }
-
-    #[test]
-    fn test_format_infobox_from_used_properties() {
-        assert_eq!(
-            format_infobox_from_used_properties(
-                "Earth".to_string(),
-                vec!["P31".to_string(), "P361".to_string(), "P571".to_string()]
-            ),
-            concat![
-                "┌───────┐\n",
-                "│ Earth │\n", // item identifier: Q2
-                "├───────┤\n",
-                "│ P31   │\n", // property: instance of
-                "│ P361  │\n", // property: part of
-                "│ P571  │\n", // property: inception
-                "└───────┘\n"
+                18, 154, 170, 178, 275, 277, 306, 348, 400, 548,
+                571, 577, 856, 1324, 2096
             ]
         );
     }

src/main.rs

@@ -1,6 +1,13 @@
-use structopt::StructOpt;
 use wikipedia_infobox_analyzer::wikipedia_infobox_analyzer::*;
 
+use ascii_table::{Align, AsciiTable};
+use dirs_next::cache_dir;
+use std::collections::HashMap;
+use std::fs;
+use std::io;
+use std::path::PathBuf;
+use structopt::StructOpt;
+
 #[derive(StructOpt)]
 struct Cli {
     #[structopt(default_value = "Earth", long = "title")]
@@ -11,18 +18,227 @@ struct Cli {
     infobox_template_file: std::path::PathBuf,
 }
 
-fn main() -> Result<(), Box<dyn std::error::Error>> {
-    let args = Cli::from_args();
+#[derive(Clone, Debug)]
+struct StorageDisk {
+    base_path: PathBuf,
+}
+
+impl StorageDisk {
+    fn new(app_name: &str) -> std::io::Result<Self> {
+        // Get OS-specific cache directory, otherwise use a fallback location
+        let mut path = match cache_dir() {
+            Some(path) => path,
+            None => PathBuf::from("./.cache"),
+        };
+
+        path.push(app_name);
+        match fs::create_dir_all(&path) {
+            Ok(_) => Ok(StorageDisk { base_path: path }),
+            Err(_) => panic!("Failed to create storage directory: {path:?}"),
+        }
+    }
+
+    fn store_data(&self, filename: String, data: String) -> std::io::Result<()> {
+        let file_path = self.base_path.join(filename);
+        fs::write(file_path, data)
+    }
+
+    fn read_data(&self, filename: String) -> std::io::Result<String> {
+        let file_path = self.base_path.join(filename);
+        fs::read_to_string(file_path)
+    }
+}
+
+struct Cache {
+    file_name: String,
+    storage: StorageDisk,
+    property_labels: HashMap<String, String>,
+}
+
+impl Cache {
+    fn new(file_name: String, storage: StorageDisk) -> std::io::Result<Self> {
+        let property_labels: HashMap<String, String> = match storage.read_data(file_name.clone()) {
+            Ok(raw_data) => {
+                serde_json::from_str(&raw_data)?
+            }
+            Err(err) if err.kind() == io::ErrorKind::NotFound => {
+                println!("No cache found, starting an empty cache");
+
+                HashMap::new()
+            }
+            Err(err) => {
+                println!("Unknown error occurred, exiting gracefully");
+                return Err(err)
+            }
+        };
+
+        let cache = Cache {
+            file_name,
+            storage,
+            property_labels
+        };
+        cache.sync_to_storage()?;
+
+        Ok(cache)
+    }
+
+    fn sync_to_storage(&self) -> std::io::Result<()> {
+        let data = serde_json::to_string(&self.property_labels)?;
+        self.storage.store_data(self.file_name.clone(), data)?;
+        Ok(())
+    }
+
+    #[allow(dead_code)]
+    fn delete(&mut self) -> std::io::Result<()> {
+        fs::remove_file(self.storage.base_path.join(self.file_name.clone()))?;
+        self.property_labels = HashMap::new();
+        Ok(())
+    }
+
+    async fn lookup_property_name(&mut self, pid: u64) -> std::io::Result<String> {
+        let key = format!("P{pid}");
+        if let Some(name) = self.property_labels.get(&key) {
+            return Ok(name.to_string());
+        }
+
+        // If the property name cannot be found, fetch the name
+        let name = fetch_name_for_wiki_property(pid).await;
+
+        // Update cache with new value, so it can be used the next time
+        self.property_labels.insert(key, name.clone());
+        self.sync_to_storage()?;
+
+        Ok(name)
+    }
+}
+
+/// Creates an ascii table out of the properties
+fn format_property_usage(name: String, qid: String, properties_template: Vec<u64>, properties_item: Vec<u64>, properties_labels: Vec<String>) -> String {
+    let mut ascii_table = AsciiTable::default();
+    ascii_table.set_max_width(80);
+    ascii_table
+        .column(0)
+        .set_header("Infobox requires")
+        .set_align(Align::Left);
+
+    ascii_table
+        .column(1)
+        .set_header(format!("Wikidata {name} ({qid})"))
+        .set_align(Align::Left);
+
+    ascii_table
+        .column(2)
+        .set_header("Property name")
+        .set_align(Align::Left);
+
+    // Display the property columns as a table
+    let data: Vec<Vec<String>> = join_columns(&properties_template, &properties_item)
+        .iter()
+        .enumerate()
+        .map(|(index, &m)| {
+              let (v1, v2) = m;
+              let property_name = if index >= properties_template.len() {
+                  "".to_string()
+              } else {
+                  properties_labels[index].clone()
+              };
+              vec![ v1.map_or("".to_owned(), |v| format!("P{v}").to_owned())
+                  , v2.map_or("".to_owned(), |v| format!("P{v}").to_owned())
+                  , property_name]
+            }
+        ).collect();
+    ascii_table.format(data)
+}
+
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    // Initialize cache and storage
+    let storage = StorageDisk::new("wikipedia-infobox-analyzer")?;
+    let mut cache = Cache::new("property_labels.cache".to_string(), storage)?;
 
     // Input to the program
-    let qid = fetch_wiki_item_by_article_title(
-        args.wikipedia_article_title,
-        args.wikipedia_language_code,
-    );
-    let _template_file = std::fs::read_to_string(&args.infobox_template_file)?;
+    let args = Cli::from_args();
+    let title = args.wikipedia_article_title;
+    let language = args.wikipedia_language_code;
+    let template = std::fs::read_to_string(&args.infobox_template_file)?;
 
-    println!("Qid: {qid}");
-    println!("Template file exists");
+    let qid = fetch_wiki_item_by_article_title(title.clone(), language.clone());
+    let properties_template = extract_used_properties_from_template(template);
+    let properties_item = fetch_properties_for_wiki_item(qid);
+    let mut properties_labels: Vec<String> = vec![];
+
+    for property in &properties_template {
+        let name = cache.lookup_property_name(*property).await?;
+        properties_labels.push(name);
+    }
+
+    let table = format_property_usage(
+        title.clone(),
+        format!("Q{qid}"),
+        properties_template,
+        properties_item,
+        properties_labels,
+    );
+
+    // Output of the program
+    println!("{table}");
 
     Ok(())
 }
+
+#[cfg(test)]
+mod tests_main {
+    use super::*;
+
+    #[tokio::test]
+    async fn test_cache() {
+        let storage = StorageDisk::new("wikipedia-infobox-analyzer").unwrap();
+        let mut cache = Cache::new("property_labels.test.cache".to_string(), storage.clone()).unwrap();
+        let _ = cache.delete();
+
+        // Cache should be clear after deletion
+        assert_eq!(cache.property_labels, HashMap::new());
+
+        let name = cache.lookup_property_name(31).await.unwrap();
+        assert_eq!(name, "instance of");
+
+        // Cache should store values that are looked up
+        assert_eq!(cache.property_labels.get("P31").unwrap(), "instance of");
+
+        // Cache should be persistent if the same file is passed
+        let mut cache_new = Cache::new("property_labels.test.cache".to_string(), storage.clone()).unwrap();
+        assert_eq!(cache_new.property_labels.get("P31").unwrap(), "instance of");
+
+        // Cleanup
+        let _ = cache_new.delete();
+    }
+
+    #[test]
+    fn test_format_property_usage() {
+        assert_eq!(
+            format_property_usage(
+                "Earth".to_string(),
+                "Q2".to_string(),
+                vec![18, 170, 571],
+                vec![18, 31, 138, 170, 361, 571],
+                vec![ "image".to_string()
+                    , "creator".to_string()
+                    , "inception".to_string()
+                    ]
+            ),
+
+            concat![
+                "┌──────────────────┬─────────────────────┬───────────────┐\n",
+                "│ Infobox requires │ Wikidata Earth (Q2) │ Property name │\n",
+                "├──────────────────┼─────────────────────┼───────────────┤\n",
+                "│ P18              │ P18                 │ image         │\n",
+                "│ P170             │ P170                │ creator       │\n",
+                "│ P571             │ P571                │ inception     │\n",
+                "│                  │ P31                 │               │\n",
+                "│                  │ P138                │               │\n",
+                "│                  │ P361                │               │\n",
+                "└──────────────────┴─────────────────────┴───────────────┘\n",
+            ]
+        );
+    }
+}