Tool-using mice - Lockbox#

Dataset from Reiske et al. [2026], containing video and pose files of individual mice solving mechanical puzzle “lockboxes” recorded from three camera perspectives (top, front, side).

../_images/lockbox1.png

From Fig. 1 in Reiske et al. [2026].

import zipfile
from pathlib import Path

import pandas as pd
import requests
import xarray as xr
from movement.io import load_poses, save_poses
from movement.kinematics import compute_speed, compute_velocity
from movement.utils.vector import compute_norm

import ethograph as eto
from ethograph.io.pairing import pair_media

Download data#

def download_and_extract(url: str, data_folder: Path) -> None:
    zip_path = data_folder / "dataset.zip"
    data_folder.mkdir(parents=True, exist_ok=True)
    response = requests.get(url, stream=True)
    response.raise_for_status()
    with open(zip_path, "wb") as f:
        for chunk in response.iter_content(chunk_size=8192):
            f.write(chunk)
    with zipfile.ZipFile(zip_path, "r") as zip_ref:
        zip_ref.extractall(data_folder)
    zip_path.unlink()


try:
    _here = Path(__vsc_ipynb_file__).parent
except NameError:
    _here = Path().resolve()

data_folder = _here.parent / "data" / "lockbox"
url = "https://www.dropbox.com/scl/fo/h7nkai8574h23qfq9m1b2/AP4gNZOpDJJ7z0yGtbWQiOc?rlkey=w36jzxqjkghg0j0xva5zsxy2v&e=1&st=5r9msqjw&dl=1"
download_and_extract(url, data_folder)

data_folder = data_folder / "labeled"

Build feature datasets#

Convert each trial’s DeepLabCut .h5 tracks to CSV (so the GUI and the alignment cell below can reference them), compute kinematic features, and save the merged lockbox.nc.

fps = 30
cameras = ["front-view", "side-view", "top-down-view"]

trials = [
    "2021-02-15_07-32-44_segment1_mouse324",
    # "2021-03-05_08-36-42_segment1_mouse324",
    # "2021-05-24_07-36-05_segment1_mouse291",
    # "2021-05-25_08-19-50_segment2_mouse291",
    "2021-05-31_07-34-21_segment2_mouse291",
    "2021-05-31_07-34-21_segment3_mouse291",
]

ds_list = []

for trial in trials:
    trial_datasets: dict[str, xr.Dataset] = {}

    for file in sorted(data_folder.glob(f"{trial}*-tracks.h5")):
        name = file.name

        df = pd.read_hdf(file)
        ds = load_poses.from_dlc_style_df(df, fps=fps)

        # Save CSV so the GUI and the alignment cell below can reference the tracks.
        csv_path = str(file).replace(".h5", ".csv")
        save_poses.to_dlc_file(ds, csv_path)

        if "front-view" in name:
            ds["front_velocity"] = compute_velocity(ds.position)
            ds["front_speed"] = compute_speed(ds.position)
            trial_datasets["front-view"] = ds

        elif "top-down-view" in name:
            head_centre_pos = ds.position.sel(keypoint=["ear_left", "ear_right"]).mean("keypoint")
            ds["topview_distance_head_lever_tip"] = compute_norm(
                ds.position.sel(keypoint="lever_tip") - head_centre_pos
            )
            ds["topview_distance_head_stick_head"] = compute_norm(
                ds.position.sel(keypoint="stick_head") - head_centre_pos
            )
            ds["topview_distance_head_ball"] = compute_norm(ds.position.sel(keypoint="ball") - head_centre_pos)
            trial_datasets["top-down-view"] = ds

        elif "side-view" in name:
            trial_datasets["side-view"] = ds

    if not trial_datasets:
        continue

    ds_merged = xr.merge(trial_datasets.values(), compat="override")
    if "top-down-view" in trial_datasets:
        ds_merged["position"] = trial_datasets["top-down-view"]["position"]
    ds_merged.attrs["trial"] = trial
    ds_list.append(ds_merged)

dt = eto.from_datasets(ds_list)
dt.save(data_folder / "lockbox.nc")
print(f"Saved to {data_folder / 'lockbox.nc'}")

Saved to c:\Users\Admin\Documents\Akseli\Code\ethograph\data\lockbox\labeled\lockbox.nc

Build NWB alignment#

Discover the videos and the pose CSVs written above for each trial/camera and write the per-trial alignment.nwb. Trial durations are inferred from the media (no start_time/stop_time needed).

# Filenames carry a "_<object>_" infix (ball / sliding-door / stick / lever)
# between the trial id and the camera, so discover media by glob, not exact name.
video_by_trial: dict[str, dict[str, str]] = {}
pose_by_trial: dict[str, dict[str, str]] = {}

for trial in trials:
    for cam in cameras:
        videos = sorted(data_folder.glob(f"{trial}_*{cam}.avi"))
        if videos:
            video_by_trial.setdefault(trial, {})[cam] = str(videos[0])

        # CSV tracks written by the feature cell above (movement appends _individual_<id>).
        poses = sorted(data_folder.glob(f"{trial}_*{cam}-tracks*.csv"))
        if poses:
            pose_by_trial.setdefault(trial, {})[cam] = str(poses[0])

session_table = pd.DataFrame({"trial": trials})
for cam in cameras:
    session_table[f"video_{cam}"] = [video_by_trial.get(t, {}).get(cam, "") for t in trials]
    session_table[f"pose_{cam}"] = [pose_by_trial.get(t, {}).get(cam, "") for t in trials]
session_table = session_table.loc[:, (session_table != "").any()]

nwb_path = data_folder / ".ethograph" / "alignment.nwb"
pair_media(
    trial_table=session_table,
    stream_rates={"video": float(fps), "pose": float(fps)},
    output_path=nwb_path,
    pose_fps=float(fps),
)
print(f"Saved alignment to {nwb_path}")

Saved alignment to c:\Users\Admin\Documents\Akseli\Code\ethograph\data\lockbox\labeled\.ethograph\alignment.nwb