diff --git a/modelling/data/cluster_analysis.json b/modelling/data/cluster_analysis.json index e8981cc..d397994 100644 --- a/modelling/data/cluster_analysis.json +++ b/modelling/data/cluster_analysis.json @@ -1,6 +1,6 @@ { - "schema_version": "species-similarity-clusters/v1", - "created_utc": "2026-05-11T16:16:44.060762+00:00", + "schema_version": "species-similarity-clusters/v2", + "created_utc": "2026-05-12T09:48:39.526449+00:00", "source_similarity_schema_version": "species-similarity/v1", "source_feature_schema_version": "species-feature-table/v1", "n_species": 39, @@ -56,6 +56,1411 @@ "Daisy", "Song Thrush" ], + "linkage": { + "format": "scipy.cluster.hierarchy.linkage", + "columns": [ + "left_child", + "right_child", + "distance", + "n_leaves" + ], + "node_id_convention": "Leaf nodes are 0..n_species-1 in species_input_order; internal nodes are n_species..2*n_species-2 in linkage row order.", + "species_input_order": [ + "Blackbird", + "Blue Tit", + "Common Cleavers", + "Common Starling", + "Daisy", + "Dunnock", + "Goldfinch", + "Great Tit", + "House Sparrow", + "Jay", + "Magpie", + "Mute Swan", + "Robin", + "Shepherds Purse", + "Skylark", + "Song Thrush", + "Woodpigeon", + "Wren", + "Bluebell", + "Brimstone Butterfly", + "Buttercup", + "Chiffchaff", + "Common Poppy", + "Cow Parsley", + "Cowslip", + "Cuckoo Pint", + "Dandelion", + "Garlic Mustard", + "Orange Tip Butterfly", + "Peacock Butterfly", + "Red Admiral Butterfly", + "Red Campion", + "Red Dead Nettle", + "Rosebay Willowherb", + "Snowdrop", + "Speckled Wood Butterfly", + "Swallow", + "Swift", + "Redwing" + ], + "leaf_order_indices": [ + 38, + 26, + 19, + 32, + 31, + 20, + 36, + 21, + 29, + 25, + 22, + 37, + 30, + 35, + 33, + 34, + 18, + 27, + 23, + 24, + 28, + 9, + 11, + 12, + 0, + 6, + 16, + 3, + 10, + 8, + 2, + 13, + 14, + 17, + 1, + 5, + 7, + 4, + 15 + ], + "leaf_order_species": [ + "Redwing", + "Dandelion", + "Brimstone Butterfly", + "Red Dead Nettle", + "Red Campion", + "Buttercup", + "Swallow", + "Chiffchaff", + "Peacock Butterfly", + "Cuckoo Pint", + "Common Poppy", + "Swift", + "Red Admiral Butterfly", + "Speckled Wood Butterfly", + "Rosebay Willowherb", + "Snowdrop", + "Bluebell", + "Garlic Mustard", + "Cow Parsley", + "Cowslip", + "Orange Tip Butterfly", + "Jay", + "Mute Swan", + "Robin", + "Blackbird", + "Goldfinch", + "Woodpigeon", + "Common Starling", + "Magpie", + "House Sparrow", + "Common Cleavers", + "Shepherds Purse", + "Skylark", + "Wren", + "Blue Tit", + "Dunnock", + "Great Tit", + "Daisy", + "Song Thrush" + ], + "matrix": [ + [ + 24, + 28, + 0.039297, + 2 + ], + [ + 22, + 37, + 0.042911, + 2 + ], + [ + 20, + 36, + 0.051449, + 2 + ], + [ + 19, + 32, + 0.081764, + 2 + ], + [ + 3, + 10, + 0.102899, + 2 + ], + [ + 1, + 5, + 0.122013, + 2 + ], + [ + 13, + 14, + 0.125027, + 2 + ], + [ + 4, + 15, + 0.137596, + 2 + ], + [ + 0, + 6, + 0.149372, + 2 + ], + [ + 23, + 39, + 0.152592, + 3 + ], + [ + 7, + 46, + 0.152705, + 3 + ], + [ + 26, + 42, + 0.159392, + 3 + ], + [ + 2, + 45, + 0.163416, + 3 + ], + [ + 31, + 41, + 0.16918, + 3 + ], + [ + 11, + 12, + 0.173445, + 2 + ], + [ + 44, + 49, + 0.18317, + 5 + ], + [ + 27, + 48, + 0.183288, + 4 + ], + [ + 50, + 52, + 0.190224, + 6 + ], + [ + 21, + 29, + 0.196919, + 2 + ], + [ + 16, + 43, + 0.205412, + 3 + ], + [ + 17, + 54, + 0.217734, + 6 + ], + [ + 47, + 58, + 0.224006, + 5 + ], + [ + 25, + 40, + 0.234117, + 3 + ], + [ + 18, + 55, + 0.242022, + 5 + ], + [ + 51, + 59, + 0.25963, + 9 + ], + [ + 30, + 35, + 0.265432, + 2 + ], + [ + 53, + 60, + 0.265466, + 7 + ], + [ + 56, + 57, + 0.286738, + 8 + ], + [ + 61, + 64, + 0.308973, + 5 + ], + [ + 8, + 63, + 0.363735, + 10 + ], + [ + 66, + 67, + 0.387002, + 13 + ], + [ + 65, + 68, + 0.396456, + 17 + ], + [ + 34, + 62, + 0.417987, + 6 + ], + [ + 33, + 71, + 0.462411, + 7 + ], + [ + 69, + 72, + 0.494517, + 20 + ], + [ + 9, + 70, + 0.502428, + 18 + ], + [ + 73, + 74, + 0.648667, + 38 + ], + [ + 38, + 75, + 0.75463, + 39 + ] + ], + "merges": [ + { + "node_id": 39, + "left_child": 24, + "right_child": 28, + "distance": 0.039297, + "n_leaves": 2, + "species": [ + "Cowslip", + "Orange Tip Butterfly" + ], + "left_species": [ + "Cowslip" + ], + "right_species": [ + "Orange Tip Butterfly" + ] + }, + { + "node_id": 40, + "left_child": 22, + "right_child": 37, + "distance": 0.042911, + "n_leaves": 2, + "species": [ + "Common Poppy", + "Swift" + ], + "left_species": [ + "Common Poppy" + ], + "right_species": [ + "Swift" + ] + }, + { + "node_id": 41, + "left_child": 20, + "right_child": 36, + "distance": 0.051449, + "n_leaves": 2, + "species": [ + "Buttercup", + "Swallow" + ], + "left_species": [ + "Buttercup" + ], + "right_species": [ + "Swallow" + ] + }, + { + "node_id": 42, + "left_child": 19, + "right_child": 32, + "distance": 0.081764, + "n_leaves": 2, + "species": [ + "Brimstone Butterfly", + "Red Dead Nettle" + ], + "left_species": [ + "Brimstone Butterfly" + ], + "right_species": [ + "Red Dead Nettle" + ] + }, + { + "node_id": 43, + "left_child": 3, + "right_child": 10, + "distance": 0.102899, + "n_leaves": 2, + "species": [ + "Common Starling", + "Magpie" + ], + "left_species": [ + "Common Starling" + ], + "right_species": [ + "Magpie" + ] + }, + { + "node_id": 44, + "left_child": 1, + "right_child": 5, + "distance": 0.122013, + "n_leaves": 2, + "species": [ + "Blue Tit", + "Dunnock" + ], + "left_species": [ + "Blue Tit" + ], + "right_species": [ + "Dunnock" + ] + }, + { + "node_id": 45, + "left_child": 13, + "right_child": 14, + "distance": 0.125027, + "n_leaves": 2, + "species": [ + "Shepherds Purse", + "Skylark" + ], + "left_species": [ + "Shepherds Purse" + ], + "right_species": [ + "Skylark" + ] + }, + { + "node_id": 46, + "left_child": 4, + "right_child": 15, + "distance": 0.137596, + "n_leaves": 2, + "species": [ + "Daisy", + "Song Thrush" + ], + "left_species": [ + "Daisy" + ], + "right_species": [ + "Song Thrush" + ] + }, + { + "node_id": 47, + "left_child": 0, + "right_child": 6, + "distance": 0.149372, + "n_leaves": 2, + "species": [ + "Blackbird", + "Goldfinch" + ], + "left_species": [ + "Blackbird" + ], + "right_species": [ + "Goldfinch" + ] + }, + { + "node_id": 48, + "left_child": 23, + "right_child": 39, + "distance": 0.152592, + "n_leaves": 3, + "species": [ + "Cow Parsley", + "Cowslip", + "Orange Tip Butterfly" + ], + "left_species": [ + "Cow Parsley" + ], + "right_species": [ + "Cowslip", + "Orange Tip Butterfly" + ] + }, + { + "node_id": 49, + "left_child": 7, + "right_child": 46, + "distance": 0.152705, + "n_leaves": 3, + "species": [ + "Great Tit", + "Daisy", + "Song Thrush" + ], + "left_species": [ + "Great Tit" + ], + "right_species": [ + "Daisy", + "Song Thrush" + ] + }, + { + "node_id": 50, + "left_child": 26, + "right_child": 42, + "distance": 0.159392, + "n_leaves": 3, + "species": [ + "Dandelion", + "Brimstone Butterfly", + "Red Dead Nettle" + ], + "left_species": [ + "Dandelion" + ], + "right_species": [ + "Brimstone Butterfly", + "Red Dead Nettle" + ] + }, + { + "node_id": 51, + "left_child": 2, + "right_child": 45, + "distance": 0.163416, + "n_leaves": 3, + "species": [ + "Common Cleavers", + "Shepherds Purse", + "Skylark" + ], + "left_species": [ + "Common Cleavers" + ], + "right_species": [ + "Shepherds Purse", + "Skylark" + ] + }, + { + "node_id": 52, + "left_child": 31, + "right_child": 41, + "distance": 0.16918, + "n_leaves": 3, + "species": [ + "Red Campion", + "Buttercup", + "Swallow" + ], + "left_species": [ + "Red Campion" + ], + "right_species": [ + "Buttercup", + "Swallow" + ] + }, + { + "node_id": 53, + "left_child": 11, + "right_child": 12, + "distance": 0.173445, + "n_leaves": 2, + "species": [ + "Mute Swan", + "Robin" + ], + "left_species": [ + "Mute Swan" + ], + "right_species": [ + "Robin" + ] + }, + { + "node_id": 54, + "left_child": 44, + "right_child": 49, + "distance": 0.18317, + "n_leaves": 5, + "species": [ + "Blue Tit", + "Dunnock", + "Great Tit", + "Daisy", + "Song Thrush" + ], + "left_species": [ + "Blue Tit", + "Dunnock" + ], + "right_species": [ + "Great Tit", + "Daisy", + "Song Thrush" + ] + }, + { + "node_id": 55, + "left_child": 27, + "right_child": 48, + "distance": 0.183288, + "n_leaves": 4, + "species": [ + "Garlic Mustard", + "Cow Parsley", + "Cowslip", + "Orange Tip Butterfly" + ], + "left_species": [ + "Garlic Mustard" + ], + "right_species": [ + "Cow Parsley", + "Cowslip", + "Orange Tip Butterfly" + ] + }, + { + "node_id": 56, + "left_child": 50, + "right_child": 52, + "distance": 0.190224, + "n_leaves": 6, + "species": [ + "Dandelion", + "Brimstone Butterfly", + "Red Dead Nettle", + "Red Campion", + "Buttercup", + "Swallow" + ], + "left_species": [ + "Dandelion", + "Brimstone Butterfly", + "Red Dead Nettle" + ], + "right_species": [ + "Red Campion", + "Buttercup", + "Swallow" + ] + }, + { + "node_id": 57, + "left_child": 21, + "right_child": 29, + "distance": 0.196919, + "n_leaves": 2, + "species": [ + "Chiffchaff", + "Peacock Butterfly" + ], + "left_species": [ + "Chiffchaff" + ], + "right_species": [ + "Peacock Butterfly" + ] + }, + { + "node_id": 58, + "left_child": 16, + "right_child": 43, + "distance": 0.205412, + "n_leaves": 3, + "species": [ + "Woodpigeon", + "Common Starling", + "Magpie" + ], + "left_species": [ + "Woodpigeon" + ], + "right_species": [ + "Common Starling", + "Magpie" + ] + }, + { + "node_id": 59, + "left_child": 17, + "right_child": 54, + "distance": 0.217734, + "n_leaves": 6, + "species": [ + "Wren", + "Blue Tit", + "Dunnock", + "Great Tit", + "Daisy", + "Song Thrush" + ], + "left_species": [ + "Wren" + ], + "right_species": [ + "Blue Tit", + "Dunnock", + "Great Tit", + "Daisy", + "Song Thrush" + ] + }, + { + "node_id": 60, + "left_child": 47, + "right_child": 58, + "distance": 0.224006, + "n_leaves": 5, + "species": [ + "Blackbird", + "Goldfinch", + "Woodpigeon", + "Common Starling", + "Magpie" + ], + "left_species": [ + "Blackbird", + "Goldfinch" + ], + "right_species": [ + "Woodpigeon", + "Common Starling", + "Magpie" + ] + }, + { + "node_id": 61, + "left_child": 25, + "right_child": 40, + "distance": 0.234117, + "n_leaves": 3, + "species": [ + "Cuckoo Pint", + "Common Poppy", + "Swift" + ], + "left_species": [ + "Cuckoo Pint" + ], + "right_species": [ + "Common Poppy", + "Swift" + ] + }, + { + "node_id": 62, + "left_child": 18, + "right_child": 55, + "distance": 0.242022, + "n_leaves": 5, + "species": [ + "Bluebell", + "Garlic Mustard", + "Cow Parsley", + "Cowslip", + "Orange Tip Butterfly" + ], + "left_species": [ + "Bluebell" + ], + "right_species": [ + "Garlic Mustard", + "Cow Parsley", + "Cowslip", + "Orange Tip Butterfly" + ] + }, + { + "node_id": 63, + "left_child": 51, + "right_child": 59, + "distance": 0.25963, + "n_leaves": 9, + "species": [ + "Common Cleavers", + "Shepherds Purse", + "Skylark", + "Wren", + "Blue Tit", + "Dunnock", + "Great Tit", + "Daisy", + "Song Thrush" + ], + "left_species": [ + "Common Cleavers", + "Shepherds Purse", + "Skylark" + ], + "right_species": [ + "Wren", + "Blue Tit", + "Dunnock", + "Great Tit", + "Daisy", + "Song Thrush" + ] + }, + { + "node_id": 64, + "left_child": 30, + "right_child": 35, + "distance": 0.265432, + "n_leaves": 2, + "species": [ + "Red Admiral Butterfly", + "Speckled Wood Butterfly" + ], + "left_species": [ + "Red Admiral Butterfly" + ], + "right_species": [ + "Speckled Wood Butterfly" + ] + }, + { + "node_id": 65, + "left_child": 53, + "right_child": 60, + "distance": 0.265466, + "n_leaves": 7, + "species": [ + "Mute Swan", + "Robin", + "Blackbird", + "Goldfinch", + "Woodpigeon", + "Common Starling", + "Magpie" + ], + "left_species": [ + "Mute Swan", + "Robin" + ], + "right_species": [ + "Blackbird", + "Goldfinch", + "Woodpigeon", + "Common Starling", + "Magpie" + ] + }, + { + "node_id": 66, + "left_child": 56, + "right_child": 57, + "distance": 0.286738, + "n_leaves": 8, + "species": [ + "Dandelion", + "Brimstone Butterfly", + "Red Dead Nettle", + "Red Campion", + "Buttercup", + "Swallow", + "Chiffchaff", + "Peacock Butterfly" + ], + "left_species": [ + "Dandelion", + "Brimstone Butterfly", + "Red Dead Nettle", + "Red Campion", + "Buttercup", + "Swallow" + ], + "right_species": [ + "Chiffchaff", + "Peacock Butterfly" + ] + }, + { + "node_id": 67, + "left_child": 61, + "right_child": 64, + "distance": 0.308973, + "n_leaves": 5, + "species": [ + "Cuckoo Pint", + "Common Poppy", + "Swift", + "Red Admiral Butterfly", + "Speckled Wood Butterfly" + ], + "left_species": [ + "Cuckoo Pint", + "Common Poppy", + "Swift" + ], + "right_species": [ + "Red Admiral Butterfly", + "Speckled Wood Butterfly" + ] + }, + { + "node_id": 68, + "left_child": 8, + "right_child": 63, + "distance": 0.363735, + "n_leaves": 10, + "species": [ + "House Sparrow", + "Common Cleavers", + "Shepherds Purse", + "Skylark", + "Wren", + "Blue Tit", + "Dunnock", + "Great Tit", + "Daisy", + "Song Thrush" + ], + "left_species": [ + "House Sparrow" + ], + "right_species": [ + "Common Cleavers", + "Shepherds Purse", + "Skylark", + "Wren", + "Blue Tit", + "Dunnock", + "Great Tit", + "Daisy", + "Song Thrush" + ] + }, + { + "node_id": 69, + "left_child": 66, + "right_child": 67, + "distance": 0.387002, + "n_leaves": 13, + "species": [ + "Dandelion", + "Brimstone Butterfly", + "Red Dead Nettle", + "Red Campion", + "Buttercup", + "Swallow", + "Chiffchaff", + "Peacock Butterfly", + "Cuckoo Pint", + "Common Poppy", + "Swift", + "Red Admiral Butterfly", + "Speckled Wood Butterfly" + ], + "left_species": [ + "Dandelion", + "Brimstone Butterfly", + "Red Dead Nettle", + "Red Campion", + "Buttercup", + "Swallow", + "Chiffchaff", + "Peacock Butterfly" + ], + "right_species": [ + "Cuckoo Pint", + "Common Poppy", + "Swift", + "Red Admiral Butterfly", + "Speckled Wood Butterfly" + ] + }, + { + "node_id": 70, + "left_child": 65, + "right_child": 68, + "distance": 0.396456, + "n_leaves": 17, + "species": [ + "Mute Swan", + "Robin", + "Blackbird", + "Goldfinch", + "Woodpigeon", + "Common Starling", + "Magpie", + "House Sparrow", + "Common Cleavers", + "Shepherds Purse", + "Skylark", + "Wren", + "Blue Tit", + "Dunnock", + "Great Tit", + "Daisy", + "Song Thrush" + ], + "left_species": [ + "Mute Swan", + "Robin", + "Blackbird", + "Goldfinch", + "Woodpigeon", + "Common Starling", + "Magpie" + ], + "right_species": [ + "House Sparrow", + "Common Cleavers", + "Shepherds Purse", + "Skylark", + "Wren", + "Blue Tit", + "Dunnock", + "Great Tit", + "Daisy", + "Song Thrush" + ] + }, + { + "node_id": 71, + "left_child": 34, + "right_child": 62, + "distance": 0.417987, + "n_leaves": 6, + "species": [ + "Snowdrop", + "Bluebell", + "Garlic Mustard", + "Cow Parsley", + "Cowslip", + "Orange Tip Butterfly" + ], + "left_species": [ + "Snowdrop" + ], + "right_species": [ + "Bluebell", + "Garlic Mustard", + "Cow Parsley", + "Cowslip", + "Orange Tip Butterfly" + ] + }, + { + "node_id": 72, + "left_child": 33, + "right_child": 71, + "distance": 0.462411, + "n_leaves": 7, + "species": [ + "Rosebay Willowherb", + "Snowdrop", + "Bluebell", + "Garlic Mustard", + "Cow Parsley", + "Cowslip", + "Orange Tip Butterfly" + ], + "left_species": [ + "Rosebay Willowherb" + ], + "right_species": [ + "Snowdrop", + "Bluebell", + "Garlic Mustard", + "Cow Parsley", + "Cowslip", + "Orange Tip Butterfly" + ] + }, + { + "node_id": 73, + "left_child": 69, + "right_child": 72, + "distance": 0.494517, + "n_leaves": 20, + "species": [ + "Dandelion", + "Brimstone Butterfly", + "Red Dead Nettle", + "Red Campion", + "Buttercup", + "Swallow", + "Chiffchaff", + "Peacock Butterfly", + "Cuckoo Pint", + "Common Poppy", + "Swift", + "Red Admiral Butterfly", + "Speckled Wood Butterfly", + "Rosebay Willowherb", + "Snowdrop", + "Bluebell", + "Garlic Mustard", + "Cow Parsley", + "Cowslip", + "Orange Tip Butterfly" + ], + "left_species": [ + "Dandelion", + "Brimstone Butterfly", + "Red Dead Nettle", + "Red Campion", + "Buttercup", + "Swallow", + "Chiffchaff", + "Peacock Butterfly", + "Cuckoo Pint", + "Common Poppy", + "Swift", + "Red Admiral Butterfly", + "Speckled Wood Butterfly" + ], + "right_species": [ + "Rosebay Willowherb", + "Snowdrop", + "Bluebell", + "Garlic Mustard", + "Cow Parsley", + "Cowslip", + "Orange Tip Butterfly" + ] + }, + { + "node_id": 74, + "left_child": 9, + "right_child": 70, + "distance": 0.502428, + "n_leaves": 18, + "species": [ + "Jay", + "Mute Swan", + "Robin", + "Blackbird", + "Goldfinch", + "Woodpigeon", + "Common Starling", + "Magpie", + "House Sparrow", + "Common Cleavers", + "Shepherds Purse", + "Skylark", + "Wren", + "Blue Tit", + "Dunnock", + "Great Tit", + "Daisy", + "Song Thrush" + ], + "left_species": [ + "Jay" + ], + "right_species": [ + "Mute Swan", + "Robin", + "Blackbird", + "Goldfinch", + "Woodpigeon", + "Common Starling", + "Magpie", + "House Sparrow", + "Common Cleavers", + "Shepherds Purse", + "Skylark", + "Wren", + "Blue Tit", + "Dunnock", + "Great Tit", + "Daisy", + "Song Thrush" + ] + }, + { + "node_id": 75, + "left_child": 73, + "right_child": 74, + "distance": 0.648667, + "n_leaves": 38, + "species": [ + "Dandelion", + "Brimstone Butterfly", + "Red Dead Nettle", + "Red Campion", + "Buttercup", + "Swallow", + "Chiffchaff", + "Peacock Butterfly", + "Cuckoo Pint", + "Common Poppy", + "Swift", + "Red Admiral Butterfly", + "Speckled Wood Butterfly", + "Rosebay Willowherb", + "Snowdrop", + "Bluebell", + "Garlic Mustard", + "Cow Parsley", + "Cowslip", + "Orange Tip Butterfly", + "Jay", + "Mute Swan", + "Robin", + "Blackbird", + "Goldfinch", + "Woodpigeon", + "Common Starling", + "Magpie", + "House Sparrow", + "Common Cleavers", + "Shepherds Purse", + "Skylark", + "Wren", + "Blue Tit", + "Dunnock", + "Great Tit", + "Daisy", + "Song Thrush" + ], + "left_species": [ + "Dandelion", + "Brimstone Butterfly", + "Red Dead Nettle", + "Red Campion", + "Buttercup", + "Swallow", + "Chiffchaff", + "Peacock Butterfly", + "Cuckoo Pint", + "Common Poppy", + "Swift", + "Red Admiral Butterfly", + "Speckled Wood Butterfly", + "Rosebay Willowherb", + "Snowdrop", + "Bluebell", + "Garlic Mustard", + "Cow Parsley", + "Cowslip", + "Orange Tip Butterfly" + ], + "right_species": [ + "Jay", + "Mute Swan", + "Robin", + "Blackbird", + "Goldfinch", + "Woodpigeon", + "Common Starling", + "Magpie", + "House Sparrow", + "Common Cleavers", + "Shepherds Purse", + "Skylark", + "Wren", + "Blue Tit", + "Dunnock", + "Great Tit", + "Daisy", + "Song Thrush" + ] + }, + { + "node_id": 76, + "left_child": 38, + "right_child": 75, + "distance": 0.75463, + "n_leaves": 39, + "species": [ + "Redwing", + "Dandelion", + "Brimstone Butterfly", + "Red Dead Nettle", + "Red Campion", + "Buttercup", + "Swallow", + "Chiffchaff", + "Peacock Butterfly", + "Cuckoo Pint", + "Common Poppy", + "Swift", + "Red Admiral Butterfly", + "Speckled Wood Butterfly", + "Rosebay Willowherb", + "Snowdrop", + "Bluebell", + "Garlic Mustard", + "Cow Parsley", + "Cowslip", + "Orange Tip Butterfly", + "Jay", + "Mute Swan", + "Robin", + "Blackbird", + "Goldfinch", + "Woodpigeon", + "Common Starling", + "Magpie", + "House Sparrow", + "Common Cleavers", + "Shepherds Purse", + "Skylark", + "Wren", + "Blue Tit", + "Dunnock", + "Great Tit", + "Daisy", + "Song Thrush" + ], + "left_species": [ + "Redwing" + ], + "right_species": [ + "Dandelion", + "Brimstone Butterfly", + "Red Dead Nettle", + "Red Campion", + "Buttercup", + "Swallow", + "Chiffchaff", + "Peacock Butterfly", + "Cuckoo Pint", + "Common Poppy", + "Swift", + "Red Admiral Butterfly", + "Speckled Wood Butterfly", + "Rosebay Willowherb", + "Snowdrop", + "Bluebell", + "Garlic Mustard", + "Cow Parsley", + "Cowslip", + "Orange Tip Butterfly", + "Jay", + "Mute Swan", + "Robin", + "Blackbird", + "Goldfinch", + "Woodpigeon", + "Common Starling", + "Magpie", + "House Sparrow", + "Common Cleavers", + "Shepherds Purse", + "Skylark", + "Wren", + "Blue Tit", + "Dunnock", + "Great Tit", + "Daisy", + "Song Thrush" + ] + } + ] + }, "species_cluster_ids": { "Blackbird": 7, "Blue Tit": 8, @@ -104,6 +1509,7 @@ "species": [ "Redwing" ], + "description": "Single-species cluster containing Redwing, mainly representing core winter winter visitor with autumn arrival component. The defining pattern is a winter peak around January, a moderate autumn component, moderate summer suppression, and slow arrival fast departure response dynamics. Its defining traits include year wrapping winter presence, core winter winter peak, and moderate autumn component. Compared with the full species set, autumn to winter weight ratio is higher than the whole-set average and decay to growth ratio is higher than the whole-set average.", "dominant_model_family": "winter_presence", "dominant_primary_class": "winter_visitor_with_autumn_arrival_component", "numeric_summary": { @@ -349,6 +1755,7 @@ "Red Admiral Butterfly", "Speckled Wood Butterfly" ], + "description": "Cluster of 13 species, mainly representing spring extended spring seasonal presence. The fitted active window runs roughly from March to October, with a mean peak around June, and and an average width of 6.6 months. It is characterised by very broad season and moderate active window. Common high-support traits include strong offseason suppression and early peak alignment.", "dominant_model_family": "seasonal_presence", "dominant_primary_class": "extended_spring_seasonal_presence", "numeric_summary": { @@ -627,6 +2034,7 @@ "species": [ "Rosebay Willowherb" ], + "description": "Single-species cluster containing Rosebay Willowherb, mainly representing autumn moderate autumn seasonal presence. The fitted active window runs roughly from June to September, with a mean peak around September, and and an average width of 3.1 months. It is characterised by moderate season and sharp active window. Its defining traits include autumn peak, moderate season, and sharp seasonal window. Compared with the full species set, season start month is higher than the whole-set average and peak month is higher than the whole-set average.", "dominant_model_family": "seasonal_presence", "dominant_primary_class": "moderate_autumn_seasonal_presence", "numeric_summary": { @@ -850,6 +2258,7 @@ "species": [ "Snowdrop" ], + "description": "Single-species cluster containing Snowdrop, mainly representing winter narrow winter seasonal presence. The fitted active window runs roughly from February to March, with a mean peak around February, and and an average width of 1.9 months. It is characterised by narrow season and moderate active window. Its defining traits include winter peak, narrow season, and moderate seasonal window. Compared with the full species set, season midpoint month is lower than the whole-set average and season end month is lower than the whole-set average.", "dominant_model_family": "seasonal_presence", "dominant_primary_class": "narrow_winter_seasonal_presence", "numeric_summary": { @@ -1077,6 +2486,7 @@ "Cowslip", "Orange Tip Butterfly" ], + "description": "Cluster of 5 species, mainly representing spring moderate spring seasonal presence. The fitted active window runs roughly from April to June, with a mean peak around May, and and an average width of 2.3 months. It is characterised by moderate season and sharp active window. Common high-support traits include spring peak, central peak alignment, and sharp seasonal window. Compared with the full species set, fit score is lower than the whole-set average and season end month is lower than the whole-set average.", "dominant_model_family": "seasonal_presence", "dominant_primary_class": "moderate_spring_seasonal_presence", "numeric_summary": { @@ -1364,6 +2774,7 @@ "species": [ "Jay" ], + "description": "Single-species cluster containing Jay, mainly representing autumn resident with summer detectability collapse. Detectability peaks around October and and is lowest around August. The shared pattern includes weak baseline presence, moderate summer suppression, weak autumn component, and decline biased response dynamics. Its defining traits include resident detectability pattern, weak baseline presence, and autumn detectability peak. Compared with the full species set, peak month is higher than the whole-set average and target amplitude is lower than the whole-set average.", "dominant_model_family": "resident_detectability", "dominant_primary_class": "resident_with_summer_detectability_collapse", "numeric_summary": { @@ -1660,6 +3071,7 @@ "Common Starling", "Magpie" ], + "description": "Cluster of 7 species, mainly representing winter resident with spring persistence and summer suppression. Detectability peaks around February and and is lowest around September. The shared pattern includes strong baseline presence, strong summer suppression, weak autumn component, and rapid decline biased response dynamics. Common high-support traits include resident detectability pattern, meaningful year end component, and strong baseline presence. Compared with the full species set, year end to winter weight ratio is higher than the whole-set average and baseline to peak ratio is higher than the whole-set average.", "dominant_model_family": "resident_detectability", "dominant_primary_class": "resident_with_spring_persistence_and_summer_suppression", "numeric_summary": { @@ -1994,6 +3406,7 @@ "Daisy", "Song Thrush" ], + "description": "Cluster of 10 species, mainly representing spring resident with summer detectability collapse. Detectability peaks around April and and is lowest around September. The shared pattern includes weak baseline presence, moderate summer suppression, weak autumn component, and rapid decline biased response dynamics. Common high-support traits include resident detectability pattern, moderate summer suppression, and rapid decline biased response dynamics.", "dominant_model_family": "resident_detectability", "dominant_primary_class": "resident_with_summer_detectability_collapse", "numeric_summary": { diff --git a/modelling/data/cluster_dendrogram.png b/modelling/data/cluster_dendrogram.png new file mode 100644 index 0000000..1318c68 Binary files /dev/null and b/modelling/data/cluster_dendrogram.png differ diff --git a/modelling/data/cluster_summary.txt b/modelling/data/cluster_summary.txt index a204b86..cc4897a 100644 --- a/modelling/data/cluster_summary.txt +++ b/modelling/data/cluster_summary.txt @@ -4,17 +4,28 @@ Species clusters Cluster 1 --------- +Single-species cluster containing Redwing, mainly representing core winter winter visitor with autumn arrival component. The defining pattern is a winter peak around January, a moderate autumn component, moderate summer suppression, and slow arrival fast departure response dynamics. Its defining traits include year wrapping winter presence, core winter winter peak, and moderate autumn component. Compared with the full species set, autumn to winter weight ratio is higher than the whole-set average and decay to growth ratio is higher than the whole-set average. + Species (1): Redwing Dominant model family : winter_presence Dominant class : winter_visitor_with_autumn_arrival_component -Common traits : year_wrapping_winter_presence, core_winter_winter_peak, moderate_autumn_component, moderate_summer_suppression, low_baseline_presence +Common traits : year_wrapping_winter_presence (1, 100%), core_winter_winter_peak (1, 100%), moderate_autumn_component (1, 100%), moderate_summer_suppression (1, 100%), low_baseline_presence (1, 100%) Peak month mean/range : 1.00 (1.00 - 1.00) +Distinguishing numeric features: + - autumn_to_winter_weight_ratio (higher, scaled_difference=0.83) + - decay_to_growth_ratio (higher, scaled_difference=0.82) + - trough_month (lower, scaled_difference=-0.58) + - fit_score (lower, scaled_difference=-0.40) + - peak_month (lower, scaled_difference=-0.36) + Cluster 2 --------- +Cluster of 13 species, mainly representing spring extended spring seasonal presence. The fitted active window runs roughly from March to October, with a mean peak around June, and and an average width of 6.6 months. It is characterised by very broad season and moderate active window. Common high-support traits include strong offseason suppression and early peak alignment. + Species (13): Dandelion Brimstone Butterfly @@ -32,37 +43,64 @@ Species (13): Dominant model family : seasonal_presence Dominant class : extended_spring_seasonal_presence -Common traits : strong_offseason_suppression, early_peak_alignment, spring_peak, very_broad_season, moderate_seasonal_window +Common traits : strong_offseason_suppression (12, 92%), early_peak_alignment (10, 77%), spring_peak (8, 62%), very_broad_season (8, 62%), moderate_seasonal_window (7, 54%) Peak month mean/range : 5.52 (4.01 - 8.44) Season width mean : 6.55 months +Distinguishing numeric features: + - season_width_months (higher, scaled_difference=0.19) + - peak_month (higher, scaled_difference=0.14) + - season_end_month (higher, scaled_difference=0.11) + - season_midpoint_month (higher, scaled_difference=0.09) + - season_start_month (lower, scaled_difference=-0.03) + Cluster 3 --------- +Single-species cluster containing Rosebay Willowherb, mainly representing autumn moderate autumn seasonal presence. The fitted active window runs roughly from June to September, with a mean peak around September, and and an average width of 3.1 months. It is characterised by moderate season and sharp active window. Its defining traits include autumn peak, moderate season, and sharp seasonal window. Compared with the full species set, season start month is higher than the whole-set average and peak month is higher than the whole-set average. + Species (1): Rosebay Willowherb Dominant model family : seasonal_presence Dominant class : moderate_autumn_seasonal_presence -Common traits : autumn_peak, moderate_season, sharp_seasonal_window, strong_post_peak_decline, strong_offseason_suppression +Common traits : autumn_peak (1, 100%), moderate_season (1, 100%), sharp_seasonal_window (1, 100%), strong_post_peak_decline (1, 100%), strong_offseason_suppression (1, 100%) Peak month mean/range : 8.62 (8.62 - 8.62) Season width mean : 3.12 months +Distinguishing numeric features: + - season_start_month (higher, scaled_difference=0.56) + - peak_month (higher, scaled_difference=0.49) + - season_midpoint_month (higher, scaled_difference=0.29) + - season_width_months (lower, scaled_difference=-0.25) + - season_end_month (higher, scaled_difference=0.02) + Cluster 4 --------- +Single-species cluster containing Snowdrop, mainly representing winter narrow winter seasonal presence. The fitted active window runs roughly from February to March, with a mean peak around February, and and an average width of 1.9 months. It is characterised by narrow season and moderate active window. Its defining traits include winter peak, narrow season, and moderate seasonal window. Compared with the full species set, season midpoint month is lower than the whole-set average and season end month is lower than the whole-set average. + Species (1): Snowdrop Dominant model family : seasonal_presence Dominant class : narrow_winter_seasonal_presence -Common traits : winter_peak, narrow_season, moderate_seasonal_window, moderate_post_peak_decline, strong_offseason_suppression +Common traits : winter_peak (1, 100%), narrow_season (1, 100%), moderate_seasonal_window (1, 100%), moderate_post_peak_decline (1, 100%), strong_offseason_suppression (1, 100%) Peak month mean/range : 2.27 (2.27 - 2.27) Season width mean : 1.89 months +Distinguishing numeric features: + - season_midpoint_month (lower, scaled_difference=-0.71) + - season_end_month (lower, scaled_difference=-0.70) + - season_start_month (lower, scaled_difference=-0.44) + - season_width_months (lower, scaled_difference=-0.41) + - peak_month (lower, scaled_difference=-0.22) + Cluster 5 --------- +Cluster of 5 species, mainly representing spring moderate spring seasonal presence. The fitted active window runs roughly from April to June, with a mean peak around May, and and an average width of 2.3 months. It is characterised by moderate season and sharp active window. Common high-support traits include spring peak, central peak alignment, and sharp seasonal window. Compared with the full species set, fit score is lower than the whole-set average and season end month is lower than the whole-set average. + Species (5): Bluebell Garlic Mustard @@ -72,24 +110,42 @@ Species (5): Dominant model family : seasonal_presence Dominant class : moderate_spring_seasonal_presence -Common traits : spring_peak, central_peak_alignment, sharp_seasonal_window, strong_offseason_suppression, moderate_season +Common traits : spring_peak (5, 100%), central_peak_alignment (5, 100%), sharp_seasonal_window (4, 80%), strong_offseason_suppression (4, 80%), moderate_season (4, 80%) Peak month mean/range : 4.76 (4.26 - 5.29) Season width mean : 2.27 months +Distinguishing numeric features: + - fit_score (lower, scaled_difference=-0.53) + - season_end_month (lower, scaled_difference=-0.36) + - season_width_months (lower, scaled_difference=-0.36) + - season_midpoint_month (lower, scaled_difference=-0.22) + - season_start_month (higher, scaled_difference=0.07) + Cluster 6 --------- +Single-species cluster containing Jay, mainly representing autumn resident with summer detectability collapse. Detectability peaks around October and and is lowest around August. The shared pattern includes weak baseline presence, moderate summer suppression, weak autumn component, and decline biased response dynamics. Its defining traits include resident detectability pattern, weak baseline presence, and autumn detectability peak. Compared with the full species set, peak month is higher than the whole-set average and target amplitude is lower than the whole-set average. + Species (1): Jay Dominant model family : resident_detectability Dominant class : resident_with_summer_detectability_collapse -Common traits : resident_detectability_pattern, weak_baseline_presence, autumn_detectability_peak, summer_detectability_trough, weak_spring_carryover +Common traits : resident_detectability_pattern (1, 100%), weak_baseline_presence (1, 100%), autumn_detectability_peak (1, 100%), summer_detectability_trough (1, 100%), weak_spring_carryover (1, 100%) Peak month mean/range : 10.00 (10.00 - 10.00) +Distinguishing numeric features: + - peak_month (higher, scaled_difference=0.64) + - target_amplitude (lower, scaled_difference=-0.61) + - fit_score (higher, scaled_difference=0.47) + - target_mean_value (lower, scaled_difference=-0.44) + - baseline_to_peak_ratio (lower, scaled_difference=-0.31) + Cluster 7 --------- +Cluster of 7 species, mainly representing winter resident with spring persistence and summer suppression. Detectability peaks around February and and is lowest around September. The shared pattern includes strong baseline presence, strong summer suppression, weak autumn component, and rapid decline biased response dynamics. Common high-support traits include resident detectability pattern, meaningful year end component, and strong baseline presence. Compared with the full species set, year end to winter weight ratio is higher than the whole-set average and baseline to peak ratio is higher than the whole-set average. + Species (7): Mute Swan Robin @@ -101,12 +157,21 @@ Species (7): Dominant model family : resident_detectability Dominant class : resident_with_spring_persistence_and_summer_suppression -Common traits : resident_detectability_pattern, meaningful_year_end_component, strong_baseline_presence, winter_detectability_peak, weak_autumn_component +Common traits : resident_detectability_pattern (7, 100%), meaningful_year_end_component (7, 100%), strong_baseline_presence (6, 86%), winter_detectability_peak (6, 86%), weak_autumn_component (6, 86%) Peak month mean/range : 2.14 (2.00 - 3.00) +Distinguishing numeric features: + - year_end_to_winter_weight_ratio (higher, scaled_difference=0.33) + - baseline_to_peak_ratio (higher, scaled_difference=0.26) + - peak_month (lower, scaled_difference=-0.23) + - target_mean_value (higher, scaled_difference=0.23) + - decay_to_growth_ratio (lower, scaled_difference=-0.13) + Cluster 8 --------- +Cluster of 10 species, mainly representing spring resident with summer detectability collapse. Detectability peaks around April and and is lowest around September. The shared pattern includes weak baseline presence, moderate summer suppression, weak autumn component, and rapid decline biased response dynamics. Common high-support traits include resident detectability pattern, moderate summer suppression, and rapid decline biased response dynamics. + Species (10): House Sparrow Common Cleavers @@ -121,6 +186,13 @@ Species (10): Dominant model family : resident_detectability Dominant class : resident_with_summer_detectability_collapse -Common traits : resident_detectability_pattern, moderate_summer_suppression, rapid_decline_biased_response_dynamics, weak_autumn_component, meaningful_year_end_component +Common traits : resident_detectability_pattern (10, 100%), moderate_summer_suppression (10, 100%), rapid_decline_biased_response_dynamics (9, 90%), weak_autumn_component (8, 80%), meaningful_year_end_component (8, 80%) Peak month mean/range : 4.21 (3.00 - 5.00) + +Distinguishing numeric features: + - year_end_to_winter_weight_ratio (lower, scaled_difference=-0.24) + - baseline_to_peak_ratio (lower, scaled_difference=-0.15) + - target_mean_value (lower, scaled_difference=-0.12) + - autumn_to_winter_weight_ratio (lower, scaled_difference=-0.11) + - target_amplitude (higher, scaled_difference=0.05) diff --git a/modelling/data/feature_matrix.json b/modelling/data/feature_matrix.json index 5c66a7c..2eb882a 100644 --- a/modelling/data/feature_matrix.json +++ b/modelling/data/feature_matrix.json @@ -1,6 +1,6 @@ { "schema_version": "species-feature-table/v1", - "created_utc": "2026-05-11T16:16:43.300203+00:00", + "created_utc": "2026-05-12T09:48:38.767460+00:00", "description": "Whole-set seasonal ecology feature table compiled from per-species classification JSON files.", "n_species": 39, "source_files": [ diff --git a/modelling/data/species_similarity.json b/modelling/data/species_similarity.json index 02c84a7..4d335cb 100644 --- a/modelling/data/species_similarity.json +++ b/modelling/data/species_similarity.json @@ -1,8 +1,8 @@ { "schema_version": "species-similarity/v1", - "created_utc": "2026-05-11T16:16:43.309319+00:00", + "created_utc": "2026-05-12T09:48:38.777271+00:00", "source_feature_schema_version": "species-feature-table/v1", - "source_feature_created_utc": "2026-05-11T16:16:43.300203+00:00", + "source_feature_created_utc": "2026-05-12T09:48:38.767460+00:00", "n_species": 39, "top_n": 5, "method": { diff --git a/modelling/scripts/build-matrix.sh b/modelling/scripts/run-similarity.sh similarity index 95% rename from modelling/scripts/build-matrix.sh rename to modelling/scripts/run-similarity.sh index 1b04f90..202c002 100755 --- a/modelling/scripts/build-matrix.sh +++ b/modelling/scripts/run-similarity.sh @@ -47,4 +47,5 @@ python "$MODELLING_ROOT/src/feature_matrix.py" \ --similarity-summary "$MODELLING_ROOT/data/species_similarity.txt" \ --heatmap "$MODELLING_ROOT/data/species_similarity_heatmap.png" \ --clusters "$MODELLING_ROOT/data/cluster_analysis.json" \ - --cluster-summary "$MODELLING_ROOT/data/cluster_summary.txt" $WRITE_CSV + --cluster-summary "$MODELLING_ROOT/data/cluster_summary.txt" \ + --dendrogram "$MODELLING_ROOT/data/cluster_dendrogram.png"$WRITE_CSV diff --git a/modelling/src/feature_matrix.py b/modelling/src/feature_matrix.py index dabbeef..aea0ba4 100644 --- a/modelling/src/feature_matrix.py +++ b/modelling/src/feature_matrix.py @@ -7,6 +7,7 @@ from seasonal.features.similarity_heatmap import generate_species_similarity_heatmap from seasonal.features.similarity_clusters import extract_species_similarity_clusters, save_cluster_summary from seasonal.features.feature_matrix import build_feature_table, find_input_files, write_csv +from seasonal.features.similarity_dendrogram import plot_species_cluster_dendrogram from seasonal.support.console import print_error, print_message from seasonal.support.json import write_json @@ -78,6 +79,8 @@ def main() -> None: parser.add_argument("-cl", "--clusters", type=Path, required=True, help="Cluster analysis output file path") parser.add_argument("-csu", "--cluster-summary", type=Path, required=True, help="Cluster analysis summary output file path") + parser.add_argument("-d", "--dendrogram", type=Path, required=True, + help="Species similarity summary dendogram image file path") args = parser.parse_args() # Look for JSON classification files in the specified input folders @@ -118,6 +121,10 @@ def main() -> None: save_cluster_summary(clusters, args.cluster_summary) print_message(f"Species similarity text dump written to {Path(args.cluster_summary).name}") + # Generate the dendrogram + plot_species_cluster_dendrogram(clusters, args.dendrogram) + print_message(f"Species similarity dendrogram written to {Path(args.dendrogram).name}") + if __name__ == "__main__": main() diff --git a/modelling/src/seasonal/features/clustering.py b/modelling/src/seasonal/features/clustering.py index 7c33b40..489b18a 100644 --- a/modelling/src/seasonal/features/clustering.py +++ b/modelling/src/seasonal/features/clustering.py @@ -1,7 +1,7 @@ from __future__ import annotations -from typing import List, Tuple +from typing import Any, Dict, List, Sequence, Tuple import numpy as np from scipy.cluster.hierarchy import leaves_list, linkage @@ -50,3 +50,84 @@ def order_species_by_linkage(similarity_matrix: np.ndarray, linkage_method: str linkage_matrix = build_linkage_matrix(similarity_matrix, linkage_method=linkage_method) order = leaves_list(linkage_matrix).tolist() return order, linkage_matrix + + +def serialise_linkage_matrix( + linkage_matrix: np.ndarray, + species_names: Sequence[str], + leaf_order: Sequence[int] | None = None, + *, + decimals: int = 6, +) -> Dict[str, Any]: + """ + Convert a SciPy linkage matrix into a JSON-friendly dendrogram description. + + SciPy linkage rows use integer node IDs: original observations are leaves + 0..n-1, and newly merged internal nodes are n..2n-2 in row order. This + function preserves that convention so the JSON can be converted back to a + SciPy linkage matrix for plotting, while also adding species names and child + membership lists for easier inspection. + + :param linkage_matrix: SciPy linkage matrix with columns child_1, child_2, + distance and n_leaves + :param species_names: Species names in the same order used to build the + similarity matrix + :param leaf_order: Optional dendrogram leaf order returned by leaves_list + :param decimals: Number of decimal places used for stored distances + :return: JSON-serialisable linkage metadata and merge details + """ + n_species = len(species_names) + if linkage_matrix.shape != (max(n_species - 1, 0), 4): + raise ValueError( + "linkage_matrix shape does not match species_names length: " + f"shape={linkage_matrix.shape}, n_species={n_species}" + ) + + species_by_node_id: Dict[int, List[str]] = { + i: [str(name)] for i, name in enumerate(species_names) + } + + merges: List[Dict[str, Any]] = [] + scipy_rows: List[List[float]] = [] + + for row_index, row in enumerate(linkage_matrix): + left_id = int(row[0]) + right_id = int(row[1]) + distance = round(float(row[2]), decimals) + n_leaves = int(row[3]) + node_id = n_species + row_index + + left_species = species_by_node_id[left_id] + right_species = species_by_node_id[right_id] + merged_species = left_species + right_species + species_by_node_id[node_id] = merged_species + + scipy_rows.append([left_id, right_id, distance, n_leaves]) + merges.append( + { + "node_id": node_id, + "left_child": left_id, + "right_child": right_id, + "distance": distance, + "n_leaves": n_leaves, + "species": merged_species, + "left_species": left_species, + "right_species": right_species, + } + ) + + return { + "format": "scipy.cluster.hierarchy.linkage", + "columns": ["left_child", "right_child", "distance", "n_leaves"], + "node_id_convention": ( + "Leaf nodes are 0..n_species-1 in species_input_order; internal nodes " + "are n_species..2*n_species-2 in linkage row order." + ), + "species_input_order": list(species_names), + "leaf_order_indices": list(leaf_order) if leaf_order is not None else None, + "leaf_order_species": ( + [str(species_names[i]) for i in leaf_order] if leaf_order is not None else None + ), + "matrix": scipy_rows, + "merges": merges, + } diff --git a/modelling/src/seasonal/features/similarity_clusters.py b/modelling/src/seasonal/features/similarity_clusters.py index cc2260b..fd24fac 100644 --- a/modelling/src/seasonal/features/similarity_clusters.py +++ b/modelling/src/seasonal/features/similarity_clusters.py @@ -8,10 +8,10 @@ import numpy as np from scipy.cluster.hierarchy import fcluster -from seasonal.features.clustering import order_species_by_linkage +from seasonal.features.clustering import order_species_by_linkage, serialise_linkage_matrix from seasonal.features.species_similarity import build_similarity_matrix, extract_species_names from seasonal.support.numeric import round_float, safe_float -from seasonal.support.calendar import circular_month_mean, signed_circular_month_difference +from seasonal.support.calendar import circular_month_mean, signed_circular_month_difference, month_label DEFAULT_NUMERIC_FEATURES = [ @@ -55,7 +55,7 @@ } -GENERATED_SCHEMA_VERSION = "species-similarity-clusters/v1" +GENERATED_SCHEMA_VERSION = "species-similarity-clusters/v2" def extract_species_similarity_clusters( @@ -99,6 +99,11 @@ def extract_species_similarity_clusters( similarity_matrix = build_similarity_matrix(species_names, similarity_data) leaf_order, linkage_matrix = order_species_by_linkage(similarity_matrix, linkage_method=linkage_method) + linkage_details = serialise_linkage_matrix( + linkage_matrix=linkage_matrix, + species_names=species_names, + leaf_order=leaf_order, + ) if distance_threshold is not None: raw_labels = fcluster(linkage_matrix, t=distance_threshold, criterion="distance") @@ -171,6 +176,7 @@ def extract_species_similarity_clusters( "assemblages rather than fixed ecological categories.", }, "species_order": [species_names[i] for i in leaf_order], + "linkage": linkage_details, "species_cluster_ids": species_cluster_ids, "clusters": clusters, } @@ -255,10 +261,19 @@ def _summarise_cluster( top_n=top_n_distinguishing_features, ) + description = _describe_cluster( + members=members, + categorical_summary=categorical_summary, + numeric_summary=numeric_summary, + common_traits=common_traits, + distinguishing_numeric_features=distinguishing_numeric_features, + ) + return { "cluster_id": cluster_id, "n_species": len(members), "species": members, + "description": description, "dominant_model_family": _dominant_value(categorical_summary.get("model_family")), "dominant_primary_class": _dominant_value(categorical_summary.get("primary_class")), "numeric_summary": numeric_summary, @@ -273,6 +288,252 @@ def _summarise_cluster( } +def _describe_cluster( + members: Sequence[str], + categorical_summary: Dict[str, Dict[str, Any]], + numeric_summary: Dict[str, Dict[str, Any]], + common_traits: Sequence[Dict[str, Any]], + distinguishing_numeric_features: Sequence[Dict[str, Any]], +) -> str: + """ + Generate a compact human-readable interpretation of a cluster. + + The description is deliberately deterministic and evidence-based: it only + uses the cluster summaries already present in the JSON, rather than trying + to infer taxonomy or causal ecology. + """ + n_members = len(members) + model_family = _dominant_value(categorical_summary.get("model_family")) + primary_class = _dominant_value(categorical_summary.get("primary_class")) + timing = _dominant_value(categorical_summary.get("timing")) + width_class = _dominant_value(categorical_summary.get("season_width_class")) + window_shape = _dominant_value(categorical_summary.get("window_shape")) + baseline = _dominant_value(categorical_summary.get("baseline_presence")) + summer_suppression = _dominant_value(categorical_summary.get("summer_suppression")) + autumn_component = _dominant_value(categorical_summary.get("autumn_component")) + response = _dominant_value(categorical_summary.get("response_dynamics")) + + opening = _cluster_opening_sentence( + members=members, + model_family=model_family, + primary_class=primary_class, + timing=timing, + ) + + details: List[str] = [] + + peak = _summary_mean(numeric_summary, "peak_month") + start = _summary_mean(numeric_summary, "season_start_month") + end = _summary_mean(numeric_summary, "season_end_month") + width = _summary_mean(numeric_summary, "season_width_months") + trough = _summary_mean(numeric_summary, "trough_month") + + if model_family == "seasonal_presence": + seasonal_bits: List[str] = [] + if start is not None and end is not None: + seasonal_bits.append( + f"the fitted active window runs roughly from {month_label(start)} to {month_label(end)}" + ) + if peak is not None: + seasonal_bits.append(f"with a mean peak around {month_label(peak)}") + if width is not None: + seasonal_bits.append(f"and an average width of {width:.1f} months") + if seasonal_bits: + details.append(_sentence_from_bits(seasonal_bits)) + + shape_bits = [] + if width_class: + shape_bits.append(f"{_humanise_token(width_class)} season") + if window_shape: + shape_bits.append(f"{_humanise_token(window_shape)} active window") + if shape_bits: + details.append("It is characterised by " + " and ".join(shape_bits) + ".") + + elif model_family == "resident_detectability": + seasonal_bits = [] + if peak is not None: + seasonal_bits.append(f"detectability peaks around {month_label(peak)}") + if trough is not None: + seasonal_bits.append(f"and is lowest around {month_label(trough)}") + if seasonal_bits: + details.append(_sentence_from_bits(seasonal_bits)) + + dynamics_bits = [] + if baseline: + dynamics_bits.append(f"{_humanise_token(baseline)} baseline presence") + if summer_suppression: + dynamics_bits.append(f"{_humanise_token(summer_suppression)} summer suppression") + if autumn_component: + dynamics_bits.append(f"{_humanise_token(autumn_component)} autumn component") + if response: + dynamics_bits.append(f"{_humanise_token(response)} response dynamics") + if dynamics_bits: + details.append("The shared pattern includes " + _join_phrase(dynamics_bits) + ".") + + elif model_family == "winter_presence": + winter_bits = [] + if peak is not None: + winter_bits.append(f"a winter peak around {month_label(peak)}") + if autumn_component: + winter_bits.append(f"a {_humanise_token(autumn_component)} autumn component") + if summer_suppression: + winter_bits.append(f"{_humanise_token(summer_suppression)} summer suppression") + if response: + winter_bits.append(f"{_humanise_token(response)} response dynamics") + if winter_bits: + details.append("The defining pattern is " + _join_phrase(winter_bits) + ".") + + else: + if peak is not None: + details.append(f"The mean fitted peak is around {month_label(peak)}.") + + trait_phrase = _common_trait_phrase(common_traits, n_members) + if trait_phrase: + details.append(trait_phrase) + + contrast_phrase = _distinguishing_feature_phrase(distinguishing_numeric_features) + if contrast_phrase: + details.append(contrast_phrase) + + return " ".join([opening] + details) + + +def _cluster_opening_sentence( + members: Sequence[str], + model_family: Optional[str], + primary_class: Optional[str], + timing: Optional[str], +) -> str: + """ + Build the opening sentence for a human-readable cluster description + + :param members: Species names belonging to the cluster + :param model_family: Dominant model family for the cluster, if available + :param primary_class: Dominant primary classification for the cluster, if available + :param timing: Dominant timing category for the cluster, if available + :return: Opening sentence describing the cluster at a high level + """ + if len(members) == 1: + subject = f"Single-species cluster containing {members[0]}" + else: + subject = f"Cluster of {len(members)} species" + + descriptors = [] + if timing: + descriptors.append(_humanise_token(timing)) + if primary_class: + descriptors.append(_humanise_token(primary_class)) + elif model_family: + descriptors.append(_humanise_token(model_family)) + + if descriptors: + return subject + ", mainly representing " + " ".join(descriptors) + "." + return subject + "." + + +def _summary_mean(summary: Dict[str, Dict[str, Any]], feature: str) -> Optional[float]: + """ + Extract the mean value for a feature from a numeric summary dictionary + + :param summary: Numeric summary dictionary keyed by feature name + :param feature: Feature name whose mean value should be returned + :return: Mean value as a float, or None if unavailable or not numeric + """ + value = summary.get(feature, {}).get("mean") + return safe_float(value) + + +def _humanise_token(value: str) -> str: + """ + Convert an underscore-delimited token into readable text + + :param value: Machine-readable token or classification value + :return: Human-readable version of the token + """ + return value.replace("_", " ") + + +def _sentence_from_bits(bits: Sequence[str]) -> str: + """ + Join sentence fragments and format them as a sentence + + :param bits: Ordered sentence fragments to combine + :return: Capitalised sentence ending with a full stop, or an empty string + """ + if not bits: + return "" + text = _join_phrase(list(bits)) + return text[:1].upper() + text[1:] + "." + + +def _join_phrase(items: Sequence[str]) -> str: + """ + Join a sequence of phrases using readable comma-and conjunction formatting + + :param items: Phrases to combine + :return: Human-readable joined phrase + """ + items = [str(item) for item in items if item] + if not items: + return "" + if len(items) == 1: + return items[0] + if len(items) == 2: + return f"{items[0]} and {items[1]}" + return ", ".join(items[:-1]) + f", and {items[-1]}" + + +def _common_trait_phrase(common_traits: Sequence[Dict[str, Any]], n_members: int) -> Optional[str]: + """ + Build a sentence describing high-support traits shared by cluster members. + + :param common_traits: Ordered trait summary dictionaries for the cluster + :param n_members: Number of species in the cluster + :return: Human-readable trait sentence, or None if no high-support traits are present + """ + high_support_traits = [] + for row in common_traits: + trait = row.get("trait") + fraction = safe_float(row.get("fraction")) + if trait and fraction is not None and fraction >= 0.75: + high_support_traits.append(_humanise_token(str(trait))) + if len(high_support_traits) >= 3: + break + + if not high_support_traits: + return None + + if n_members == 1: + return "Its defining traits include " + _join_phrase(high_support_traits) + "." + return "Common high-support traits include " + _join_phrase(high_support_traits) + "." + + +def _distinguishing_feature_phrase(distinguishing_numeric_features: Sequence[Dict[str, Any]]) -> Optional[str]: + """ + Build a sentence describing the strongest numeric contrasts for a cluster + + :param distinguishing_numeric_features: Ordered cluster-vs-global numeric contrast dictionaries + :return: Human-readable contrast sentence, or None if no strong contrasts are present + """ + strong = [] + for row in distinguishing_numeric_features: + feature = row.get("feature") + direction = row.get("direction") + scaled = safe_float(row.get("scaled_difference")) + if not feature or direction not in {"higher", "lower"} or scaled is None: + continue + if abs(scaled) < 0.25: + continue + strong.append(f"{_humanise_token(str(feature))} is {direction} than the whole-set average") + if len(strong) >= 2: + break + + if not strong: + return None + + return "Compared with the full species set, " + _join_phrase(strong) + "." + + def _summarise_numeric_features( records: Sequence[Dict[str, Any]], feature_names: Sequence[str], @@ -506,6 +767,10 @@ def save_cluster_summary(cluster_data: dict, file_path: str) -> None: f.write(f"\nCluster {cluster_id}\n") f.write("-" * (8 + len(str(cluster_id))) + "\n") + description = cluster.get("description") + if description: + f.write(f"{description}\n\n") + f.write(f"Species ({len(species)}):\n") for name in species: f.write(f" {name}\n") @@ -528,10 +793,10 @@ def save_cluster_summary(cluster_data: dict, file_path: str) -> None: if isinstance(item, dict): label = item.get("trait") count = item.get("count") - proportion = item.get("proportion") + fraction = item.get("fraction") - if label is not None and count is not None and proportion is not None: - trait_labels.append(f"{label} ({count}, {proportion:.0%})") + if label is not None and count is not None and fraction is not None: + trait_labels.append(f"{label} ({count}, {float(fraction):.0%})") elif label is not None: trait_labels.append(str(label)) else: @@ -550,14 +815,17 @@ def save_cluster_summary(cluster_data: dict, file_path: str) -> None: if width: f.write(f"Season width mean : {width['mean']:.2f} months\n") - distinguishing = cluster.get("distinguishing_features", []) + distinguishing = cluster.get("distinguishing_numeric_features", []) if distinguishing: - f.write("\nDistinguishing features:\n") + f.write("\nDistinguishing numeric features:\n") for item in distinguishing[:5]: feature = item.get("feature") direction = item.get("direction") - strength = item.get("effect_size") + strength = item.get("scaled_difference") + + if feature is None or direction is None or strength is None: + continue - f.write(f" - {feature} ({direction}, effect={strength:.2f})\n") + f.write(f" - {feature} ({direction}, scaled_difference={float(strength):.2f})\n") diff --git a/modelling/src/seasonal/features/similarity_dendrogram.py b/modelling/src/seasonal/features/similarity_dendrogram.py new file mode 100644 index 0000000..bbc2788 --- /dev/null +++ b/modelling/src/seasonal/features/similarity_dendrogram.py @@ -0,0 +1,256 @@ +from pathlib import Path +from typing import Any, Dict +import re + +import numpy as np +import matplotlib.pyplot as plt +from matplotlib.colors import to_hex +from matplotlib.patches import Patch +from scipy.cluster.hierarchy import dendrogram + + +def _first_sentence(text: str) -> str: + """ + Extract the first sentence, excluding trainling full-stop, from a cluster description + + :param text: Full text + :return: First sentence of the text + """ + if not text: + return "" + match = re.search(r"(?<=[.!?])\s+", text.strip()) + return text.strip() if not match else text[: match.start()].strip() + + +def _get_cluster_colour( + node_id: int, + colour_clusters: bool, + species_cluster_ids: Any, + species_names: list, + node_to_leaves: dict, + cluster_colours: dict +) -> str: + """ + Given a SciPy node ID, determine the colour for that branch of the dendogram based + on the leaf nodes under it + + :param node_id: SciPy node ID + :param colour_clusters: True to colour clusters, else use black + :param species_cluster_ids: Dictionary of cluster ID membership by species + :param species_names: List of species names + :param node_to_leaves: Lookup from each SciPy node id to the leaf nodes under it + :param cluster_colours: Dictionary of cluster colours by cluster ID + """ + if not colour_clusters: + return "black" + + leaf_clusters = { + species_cluster_ids.get(species_names[i]) + for i in node_to_leaves[int(node_id)] + } + + leaf_clusters.discard(None) + + if len(leaf_clusters) == 1: + cluster_id = next(iter(leaf_clusters)) + return cluster_colours.get(cluster_id, "black") + + return "black" + + +def plot_species_cluster_dendrogram( + cluster_data: Dict[str, Any], + output_png_path: str | Path, + figsize: tuple[float, float] | None = None, + dpi: int = 180, + title: str = "Species Similarity Dendrogram", + colour_clusters: bool = True +) -> None: + + # Load the data and extract the species and cluster details + linkage_info = cluster_data.get("linkage", {}) + linkage_matrix = np.asarray(linkage_info.get("matrix"), dtype=float) + species_names = list(linkage_info.get("species_input_order", [])) + species_cluster_ids = cluster_data.get("species_cluster_ids", {}) + + if linkage_matrix.ndim != 2 or linkage_matrix.shape[1] != 4: + raise ValueError("cluster_data['linkage']['matrix'] must be an n x 4 linkage matrix") + + if linkage_matrix.shape[0] != len(species_names) - 1: + raise ValueError("Linkage matrix row count does not match species count") + + labels = [ + f"{species} [{species_cluster_ids.get(species, '?')}]" + for species in species_names + ] + + cluster_ids = sorted( + {cluster_id for cluster_id in species_cluster_ids.values() if cluster_id is not None} + ) + cmap = plt.get_cmap("tab20") + cluster_colours = { + cluster_id: to_hex(cmap(i % cmap.N)) + for i, cluster_id in enumerate(cluster_ids) + } + + # Build a lookup from each SciPy node id to the leaf indices under it. + # Leaves are 0..n_species-1; internal nodes are n_species..2*n_species-2. + n_species = len(species_names) + node_to_leaves: dict[int, set[int]] = { + i: {i} + for i in range(n_species) + } + for row_index, row in enumerate(linkage_matrix): + node_id = n_species + row_index + left_child = int(row[0]) + right_child = int(row[1]) + node_to_leaves[node_id] = ( + node_to_leaves[left_child] | node_to_leaves[right_child] + ) + + if figsize is None: + figsize = (12, max(7, len(species_names) * 0.32)) + + output_png_path = Path(output_png_path) + output_png_path.parent.mkdir(parents=True, exist_ok=True) + + fig, ax = plt.subplots(figsize=figsize) + + def link_colour_func(node_id: int) -> str: + return _get_cluster_colour(node_id, colour_clusters, species_cluster_ids, species_names, + node_to_leaves, cluster_colours) + + dendro = dendrogram( + linkage_matrix, + labels=labels, + orientation="left", + leaf_font_size=8, + color_threshold=None, + link_color_func=link_colour_func, + above_threshold_color="black", + ax=ax, + ) + + ax.set_title(title) + ax.set_xlabel("Distance: 1 - similarity") + ax.set_ylabel("Species") + ax.grid(axis="x", alpha=0.25) + + # ------------------------------------------------------------ + # Cluster span markers on the left of the plot + # ------------------------------------------------------------ + leaf_indices = dendro["leaves"] + + plotted_species = [species_names[i] for i in leaf_indices] + plotted_cluster_ids = [ + species_cluster_ids.get(species) + for species in plotted_species + ] + + # SciPy places leaves at y = 5, 15, 25, ... + y_positions = { + species: 5 + i * 10 + for i, species in enumerate(plotted_species) + } + + cluster_to_species: dict[int, list[str]] = {} + for species, cluster_id in zip(plotted_species, plotted_cluster_ids): + if cluster_id is None: + continue + cluster_to_species.setdefault(cluster_id, []).append(species) + + # x is in axes coordinates; y is in data coordinates. + # Negative x puts the markers into the left margin. + transform = ax.get_yaxis_transform() + + marker_x = -0.05 + cap_half_width = 0.018 + label_gap = 4.5 + + for cluster_id in sorted(cluster_to_species): + cluster_colour = cluster_colours.get(cluster_id, "black") + ys = [y_positions[species] for species in cluster_to_species[cluster_id]] + y_min = min(ys) - 4 + y_max = max(ys) + 4 + y_mid = (y_min + y_max) / 2 + + # Split the vertical span so the number sits in the gap. + ax.plot( + [marker_x, marker_x], + [y_min, y_mid - label_gap], + transform=transform, + clip_on=False, + linewidth=1.2, + color=cluster_colour, + ) + ax.plot( + [marker_x, marker_x], + [y_mid + label_gap, y_max], + transform=transform, + clip_on=False, + linewidth=1.2, + color=cluster_colour, + ) + + # Caps at top and bottom of the span. + ax.plot( + [marker_x - cap_half_width, marker_x + cap_half_width], + [y_min, y_min], + transform=transform, + clip_on=False, + linewidth=1.2, + color=cluster_colour, + ) + ax.plot( + [marker_x - cap_half_width, marker_x + cap_half_width], + [y_max, y_max], + transform=transform, + clip_on=False, + linewidth=1.2, + color=cluster_colour, + ) + + ax.text( + marker_x, + y_mid, + str(cluster_id), + transform=transform, + ha="center", + va="center", + fontsize=9, + fontweight="bold", + color=cluster_colour, + clip_on=False, + ) + + # ------------------------------------------------------------ + # Legend below the chart + # ------------------------------------------------------------ + + # Get the descriptions for all the clusters + cluster_descriptions = { + c["cluster_id"]: _first_sentence(c.get("description", "")) + for c in cluster_data.get("clusters", []) + } + + # Build the legend handles + legend_handles = [] + for cluster_id in sorted(cluster_to_species): + description = f"{cluster_id} : {cluster_descriptions.get(cluster_id, '')}" + legend_handles.append(Patch(facecolor=cluster_colours[cluster_id], label=description)) + + ax.legend( + handles=legend_handles, + loc="upper center", + bbox_to_anchor=(0.5, -0.08), + fontsize=8, + title_fontsize=8, + frameon=False, + ncol=1, + ) + + # Leave room on the left for cluster bars + fig.subplots_adjust(left=0.2, bottom=0.32) + + fig.savefig(output_png_path, dpi=dpi, bbox_inches="tight", pad_inches=0.25) + plt.close(fig)