Sound production in philodoptera

Stridulation in crickets (e.g. Pholidoptera littoralis) is a great animal model to study motor control/ sound production. Below is a tutorial on how we can create a trials.nc. Note that we have a video, audio and DeepLabCut pose file, meaning we can look at movement and sound features in the GUI!

Made in Slovenia, 2025.

%load_ext autoreload
%autoreload 2

import pandas as pd
from pathlib import Path
from audioio import AudioLoader
from movement.io import load_poses
from movement.kinematics import compute_velocity, compute_speed, compute_acceleration

import ethograph as eto
from ethograph.utils.download import download_example_dataset
from ethograph.io.nwb_alignment import align_media_per_trial

The autoreload extension is already loaded. To reload it, use:
  %reload_ext autoreload

Download example data

try:
    _here = Path(__vsc_ipynb_file__).parent
except NameError:
    _here = Path().resolve()

data_folder = _here.parent / "data" / "philodoptera"
download_example_dataset("philodoptera", data_folder)
print(f"\ndata_folder: {data_folder}")

Downloading philodoptera.nc... (1/4)
Downloading philodoptera.mp4... (2/4)
Downloading philodoptera.wav... (3/4)
  philodoptera.csv (4/4)

data_folder: d:\Akseli\Code\ethograph\data\philodoptera

Build NWB alignment

video_path = data_folder / "philodoptera.mp4"
audio_path = data_folder / "philodoptera.wav"
poses_csv_path = data_folder / "philodoptera.csv"
fps = 240

# Probe audio sample rate from file
with AudioLoader(str(audio_path)) as f:
    audio_sr = f.rate

# Single-session dataset: one row in the session table
session_table = pd.DataFrame({
    "trial":   ["session"],
    "video_0": [str(video_path)],
    "audio_0": [str(audio_path)],
    "pose_0":  [str(poses_csv_path)],
})

nwb_path = data_folder / ".ethograph" / "alignment.nwb"
align_media_per_trial(
    trial_table=session_table,
    stream_rates={"video": float(fps), "audio": float(audio_sr), "pose": float(fps)},
    output_path=nwb_path,
    pose_fps=float(fps),
)

Create dataset

# Load pose data and compute kinematics
ds = load_poses.from_dlc_file(poses_csv_path, fps=fps)
ds["velocity"] = compute_velocity(ds.position)
ds["speed"] = compute_speed(ds.position)
ds["acceleration"] = compute_acceleration(ds.position)

ds["individuals"] = ["Pholidoptera_littoralis_1"]
ds.attrs["Recording info"] = (
    "This cricket species Pholidoptera littoralis was recorded on a field trip "
    "in Slovenia in 2025. The video was recorded with an iPhone and the audio "
    "with a Zoom recorder. Synchronization may not be perfect."
)


ds.to_netcdf(data_folder / "philodoptera.nc")
print(f"Saved to {data_folder / 'philodoptera.nc'}")

Saved to d:\Akseli\Code\ethograph\data\philodoptera\philodoptera.nc