{ "schema": "unjournal-vowels-worked-example-v1", "status": "draft_interchange_example", "compiled_on": "2026-10-12", "scope_note": "Public-source curation. No new computational rerun, expert rating or signed nanopublication. Field names are illustrative, not validated Science Live predicates.", "sources": { "paper": "https://doi.org/10.1038/s41586-024-08106-4", "author_correction": "https://doi.org/10.1038/s41586-024-08481-y", "evaluation_summary": "https://unjournal.pubpub.org/pub/evalsumnaturalregeneration/release/3", "evaluation_1": "https://unjournal.pubpub.org/pub/e1naturalregeneration/release/1", "evaluation_2": "https://unjournal.pubpub.org/pub/e2naturalregeneration/release/1", "replication_live_writeup": "https://annefou.github.io/natural-regeneration-replication/", "replication_registry": "https://annefou.github.io/natural-regeneration-replication/published/" }, "claims": [ { "id": "paper-validation-accuracy", "text": "The paper reports 87.9% validation accuracy on 4.87 million independent random points, equally stratified by regrowth and non-regrowth.", "value": 0.879, "unit": "fraction_correct", "domain": "Original pantropical validation design", "sample_size": 4870000, "class_sampling": "equally_stratified", "source_ref": "paper", "claim_specific_expert_probability": null, "missing_probability_note": "No numerical belief about this specific accuracy claim is supplied in this example. Do not substitute the overall paper assessment." }, { "id": "paper-pantropical-area", "text": "The paper estimates 215 million hectares of potential natural regeneration across tropical forested countries and biomes.", "value": 215000000, "unit": "hectare", "source_ref": "paper", "selected_as_main_claim_by": "evaluation_1", "directly_tested_by_colombia_replication": false, "expert_claim_assessments": [ { "source_ref": "evaluation_1", "qualitative_confidence": "low", "reasoning": "Observed regrowth also reflects socioeconomic constraints; the evaluator suggests the estimate may be a lower bound on latent biophysical potential.", "numeric_probability": null, "rough_bounds_mha": { "lower": 215, "upper_note": "certainly well below 678 Mha (maximum area of opportunity in a previous study)" }, "target": "biophysical potential with anthropogenic constraints removed", "direction_vs_paper": "higher (paper's 215 Mha likely a lower bound)" }, { "source_ref": "evaluation_2", "qualitative_confidence": "low", "reasoning": "Unaddressed omission error in the input data and possible conflation of regrowth types; a rough back-of-the-envelope correction revises the estimate downwards.", "credible_range_mha": { "lower": 80, "upper": 180, "type": "rough credible range stated by the evaluator; coverage not specified" }, "target": "purely natural regeneration potential after correcting input-data errors", "direction_vs_paper": "lower", "numeric_probability": null } ], "note": "The two claim assessments point in opposite directions, partly because they target slightly different quantities." } ], "paper_assessments": [ { "id": "evaluation-1-overall-assessment", "source_ref": "evaluation_1", "published_field": "Overall assessment", "value": 50, "scale": [ 0, 100 ], "uncertainty": { "type": "subjective_credible_interval_for_overall_assessment", "coverage": 0.9, "lower": 40, "upper": 60 }, "assessment_target": "paper_overall", "updated_after_replication": false }, { "id": "evaluation-2-overall-assessment", "source_ref": "evaluation_2", "published_field": "Overall assessment", "value": 50, "scale": [ 0, 100 ], "uncertainty": { "type": "subjective_credible_interval_for_overall_assessment", "coverage": 0.9, "lower": 40, "upper": 60 }, "assessment_target": "paper_overall", "role_in_example": "optional_comparison", "revision_history": [ { "lower": 30, "upper": 80, "status": "original", "note": "Revised to 40-60 after the two evaluators discussed their reports; original kept in a footnote of the evaluation summary (release 3)." } ], "revision_source_ref": "evaluation_summary", "updated_after_replication": false } ], "concerns": [ { "id": "future-generalisation", "source_ref": "evaluation_1", "text": "Retrospective performance does not establish future prediction from currently available information.", "suggested_check": "Predict future regrowth conditional on variables available before the outcome period.", "applies_methodological_claim": [ "leakage-inflates-accuracy", "spatial-autocorrelation-inflates-accuracy" ] }, { "id": "predictor-year-leakage", "source_refs": [ "evaluation_1", "evaluation_2" ], "text": "Predictors recorded during the outcome period may incorporate information unavailable for a genuine forward prediction.", "suggested_check": "Use earlier land cover and remove predictors overlapping the outcome period.", "applies_methodological_claim": "leakage-inflates-accuracy" } ], "reported_replication_tests": [ { "id": "colombia-random-point-validation", "source_ref": "replication_live_writeup", "domain": "Colombia", "design": "Independent implementation, same kind of labels and ten biophysical predictors, random-point validation", "metric": "accuracy", "value": 0.887, "class_sampling": "balanced", "scope_note": "Similar local performance under a comparable design; not an exact reproduction of the original pantropical statistic." }, { "id": "colombia-spatial-validation", "source_ref": "replication_live_writeup", "domain": "Colombia", "design": "Spatially blocked validation", "metric": "accuracy", "result_range": [ 0.854, 0.869 ], "range_type": "results_across_validation_designs_not_confidence_interval", "investigates_concern": "future-generalisation" }, { "id": "colombia-forward-validation", "source_ref": "replication_live_writeup", "domain": "Colombia", "training_period": "2000-2012", "test_period": "2012-2024", "labels": "Independent MapBiomas regrowth labels", "class_sampling": "balanced", "metric": "balanced_accuracy", "value": 0.717, "investigates_concern": "future-generalisation", "interpretation": "Supports concern about forward generalisation in this setting. Changing labels, periods and designs prevents treating the difference as a controlled causal estimate of one failure mechanism.", "carries_out_check": "forecast-with-pre-period-predictors", "within_period_baseline_same_labels": 0.778 }, { "id": "colombia-earlier-land-cover", "source_ref": "replication_live_writeup", "domain": "Colombia", "design": "Use 1992 or 1999 land cover instead of 2000", "metric": "accuracy", "value": 0.887, "investigates_concern": "predictor-year-leakage", "carries_out_check": "earlier-land-cover-and-drop-overlapping", "values": { "land_cover_2000_baseline": 0.8867, "land_cover_1992": 0.8865, "land_cover_1999": 0.8868 } }, { "id": "colombia-overlapping-predictor-removal", "source_ref": "replication_live_writeup", "domain": "Colombia", "removed_predictors": [ "NPP", "burned area", "road density" ], "metric": "accuracy", "with_predictors": 0.897, "without_predictors": 0.893, "investigates_concern": "predictor-year-leakage", "interpretation": "These tests do not support predictor-year leakage as the explanation in this Colombia implementation; they do not establish the absence of all possible leakage.", "carries_out_check": "earlier-land-cover-and-drop-overlapping" } ], "decision_context": { "proposed_exercise_question": "What would be needed before using these results to allocate restoration funding?", "missing_inputs": [ "Additional intervention effects relative to regrowth without intervention", "Persistence of regenerated forest", "Implementation and opportunity costs", "Local feasibility" ], "status": "discussion_prompt_not_real_funding_recommendation" }, "existing_signed_records": { "citation": "https://w3id.org/sciencelive/np/RALLcB3xiCXR0UGSr7Irgo1hMohZthtGp0aCVf5qBpgoQ", "outcome": "https://w3id.org/sciencelive/np/RAZF-V9BsAmgy5a0KezH74Fd78oZy2CS6PLsizcXMTNWg", "registry_publication_date": "2026-10-04", "version_note": "Reported numerical test results above come from the live write-up consulted on 2026-10-09. Equivalence with every immutable signed record has not been checked." }, "proposed_follow_up": { "action": "Add a dated evaluation follow-up linking the relevant concern to the replication result and retain a link back to the original evaluation.", "preserve": "Original evaluation text and historical ratings", "status": "proposal_not_implemented" }, "methodological_claims": [ { "id": "leakage-inflates-accuracy", "text": "Using predictors that are affected by the outcome, or measured while it happens, makes test accuracy overstate real forecasting performance.", "references": [ "https://doi.org/10.1145/2382577.2382579", "https://doi.org/10.1016/j.patter.2023.100804" ] }, { "id": "spatial-autocorrelation-inflates-accuracy", "text": "Random test points close to training points overstate accuracy in new places; spatially blocked validation gives a more realistic estimate.", "references": [ "https://doi.org/10.1111/ecog.02881", "https://doi.org/10.1038/s41467-020-18321-y" ] } ], "suggested_checks": [ { "id": "forecast-with-pre-period-predictors", "source_ref": "evaluation_1", "resolves_concern": "future-generalisation", "text": "Formulate the prediction as a prediction of future outcomes conditional on currently observed covariates." }, { "id": "earlier-land-cover-and-drop-overlapping", "source_refs": [ "evaluation_1", "evaluation_2" ], "resolves_concern": "predictor-year-leakage", "text": "Use earlier land cover and remove predictors overlapping the outcome period." }, { "id": "compare-ml-methods", "source_ref": "evaluation_2", "resolves_concern": null, "text": "Use multiple kinds of machine-learning algorithms for at least some regions to compare sensitivity." } ], "methodological_claim_applications": [ { "methodological_claim": "leakage-inflates-accuracy", "case": "Williams et al. 2024, Colombia replication", "verdict": "Applies in principle (predictors overlap the outcome period); in Colombia, dropping them barely changes same-period accuracy, and the forward model without them still loses accuracy.", "evidence": [ "colombia-forward-validation", "colombia-earlier-land-cover", "colombia-overlapping-predictor-removal" ] } ] }