diff --git a/agent/deeptrack2-reference-to-code.zip b/agent/deeptrack2-reference-to-code.zip new file mode 100644 index 000000000..6daf58db3 Binary files /dev/null and b/agent/deeptrack2-reference-to-code.zip differ diff --git a/agent/deeptrack2-reference-to-code/SKILL.md b/agent/deeptrack2-reference-to-code/SKILL.md new file mode 100644 index 000000000..c6d4babf9 --- /dev/null +++ b/agent/deeptrack2-reference-to-code/SKILL.md @@ -0,0 +1,207 @@ +--- +name: deeptrack2-reference-to-code +description: Generate standard, runnable DeepTrack2 Python code from user descriptions or microscopy reference images, possible for training the machine learning model further. +--- + +# DeepTrack2 Reference-to-Code Skill + +## Goal + +Generate standard, runnable DeepTrack2 Python code from either: + +1. A user task description, or +2. A microscopy reference image plus a desired synthetic-data goal. + +The output should approximate the visual style or imaging modality of the reference image while following DeepTrack2 code patterns, ready for machine learning application. + +## When to use this skill + +Use this skill when the user asks to: + +- generate DeepTrack2 code +- simulate microscopy images +- create synthetic microscopy datasets +- match a microscopy modality from a reference image +- produce DeepTrack2 examples from provided documentation or code examples +- refactor or standardize DeepTrack2 code +- prepare the simulation data for machine learning model + +## Source priority + +Follow this order: + +1. Local examples in `examples/` +2. Docstring inside function/class +3. Official DeepTrack2 documentation/example +4. User-provided code or images + +Do not invent unsupported DeepTrack2 APIs. + +## Important library rules + +- Use `import deeptrack as dt` +- Prefer complete runnable Python scripts +- Use type hints where practical +- Use `if __name__ == "__main__":` +- Do not use private or undocumented APIs +- Must wrapper new operators use `dt.Lambda`, read `/examples/Lambda.py` everytime creat scripts +- Chaining as a pipeline using `>>`, `^`, and `&` as much as possible +- DeepTrack2 2.0+ does not support TensorFlow +- Do not generate TensorFlow/Keras-based DeepTrack code unless the user explicitly asks for legacy DeepTrack 1.7 code + +## General workflow + +1. Identify the user’s goal: + - particle simulation + - cell simulation + - brightfield image generation + - fluorescence image generation + - holography-like image generation + - dataset generation + - augmentation / noise modeling + - training-data pipeline + +2. Search local examples for the closest pattern. + +3. Extract: + - imports + - initialization style + - scatterer/sample construction + - optics/microscope construction + - noise and augmentation steps + - `.resolve()` or documented execution pattern + +4. Generate code in the same style as examples. + +5. Validate: + - every `dt.` appears in local examples or docs + - no fake APIs + - no TensorFlow unless legacy requested + - code is complete and runnable + +## Reference-image workflow + +When the user provides a microscopy example image: + +1. Inspect the image visually. + +2. Infer likely modality: + - brightfield + - fluorescence + - holography + - phase contrast / DIC-like + - unknown + +3. Extract visual traits: + - background brightness + - object shape + - object density + - contrast polarity + - blur / PSF size + - noise type + - illumination gradient + - field of view + - artifacts + +4. Map traits to DeepTrack2 components: + - particles/cells -> scatterers + - optics -> Brightfield / Fluorescence / holography-related examples + - blur -> optics or postprocessing + - camera noise -> Poisson or Gaussian noise examples + - variability -> random parameter distributions + - normalization -> documented preprocessing or augmentation examples + +5. Generate a runnable DeepTrack2 script that creates visually similar synthetic images. + +6. Clearly mark inferred parameters as estimates. + +## Modality mapping + +### Brightfield-like + +Visual clues: + +- gray or bright background +- darker objects +- halos or soft shadows +- uneven illumination possible + +Prefer examples involving: + +- particles or cells +- brightfield optics +- background variation +- Gaussian noise +- blur + +### Fluorescence-like + +Visual clues: + +- dark background +- bright objects +- glowing cells / spots +- shot-noise appearance + +Prefer examples involving: + +- fluorescence optics +- bright scatterers +- Poisson noise +- Gaussian blur +- intensity variation +- normalization + +### Holography-like + +Visual clues: + +- diffraction rings +- interference fringes +- phase-like contrast +- central particle with halo + +Prefer examples involving: + +- holography examples +- particle scatterers +- coherent-style optical setup +- ring-like artifacts +- background correction + +## Output format + +Return: + +1. Inferred goal or modality +2. Key visual/code assumptions +3. Complete Python code +4. Notes on parameters to calibrate +5. Caveats if the API or modality is uncertain + +## Standard simulation code style + +Generated simualtion code should follow this structure: + +```python +from __future__ import annotations + +import deeptrack as dt +import matplotlib.pyplot as plt + + +def build_pipeline(): + ... + + +def main() -> None: + pipeline = build_pipeline() + image = pipeline.resolve() + + plt.imshow(image.squeeze(), cmap="gray") + plt.axis("off") + plt.show() + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/agent/deeptrack2-reference-to-code/examples/Brightfield.py b/agent/deeptrack2-reference-to-code/examples/Brightfield.py new file mode 100644 index 000000000..1353b7901 --- /dev/null +++ b/agent/deeptrack2-reference-to-code/examples/Brightfield.py @@ -0,0 +1,56 @@ +import numpy as np +import deeptrack as dt +from matplotlib import pyplot as plt + +# ## Brightfield +# +# In `DeepTrack2`, the brightfield microscope model simulates illumination using a single wavelength of light. This is equivalent to a brightfield microscope operating with a monochromatic light source. +# +# In experimental setups however, a brightfield microscope typically uses a broad spectrum of wavelengths (white light). +# +# To achieve a more realistic simulation of white light, we will need to simulate multiple brightfield images (sampling different parts of the visible light spectrum) and then averaging the contributions of these images. +# +# We start by generating a spectrum of wavelengths and a particle scatterer object. + +IMAGE_SIZE = 150 + +# Generate a spectrum of wavelengths to sample from, 5 will be enough. +wavelengths = np.linspace(450e-9, 700e-9, 5) + +particle = dt.Sphere( + position=(IMAGE_SIZE//2, IMAGE_SIZE//4, 2) * dt.units.pixel, #put particles in the middle + radius= 0.3e-6, + position_unit="pixel", + refractive_index=1.42, +) + + +# Image the particle by sampling the spectrum. + +# Sample the wavelengths. +imaged_particle_list = [] +for wavelength in wavelengths: + # Create a brightfield microscope for a given wavelength. + single_wavelength_optics = dt.Brightfield( + resolution=1e-6, + magnification=10, + wavelength=wavelength, + padding=(32, 32, 32, 32), + output_region=(0, 0, IMAGE_SIZE, IMAGE_SIZE//2), + ) + + # Image the particle. + imaged_particle = single_wavelength_optics(particle) + + # Add background noise. + imaged_particle = imaged_particle >> dt.Gaussian(0, 0.01) + + # Append to list. + imaged_particle_list.append(imaged_particle) + +# Take the average of the images in the list. +brightfield_image = ( + sum(imaged_particle_list) / len(imaged_particle_list) +).resolve() +plt.imshow(brightfield_image, cmap="gray") +plt.show() \ No newline at end of file diff --git a/agent/deeptrack2-reference-to-code/examples/Darkfield.py b/agent/deeptrack2-reference-to-code/examples/Darkfield.py new file mode 100644 index 000000000..bf68da813 --- /dev/null +++ b/agent/deeptrack2-reference-to-code/examples/Darkfield.py @@ -0,0 +1,63 @@ +import numpy as np +import deeptrack as dt +from matplotlib import pyplot as plt + +# ## Darkfield +# We will image a particle in a darkfield modality. This requires the scatterer to be defined as a Mie scatterer object. +# + +IMAGE_SIZE = 150 + +# Generate a spectrum of wavelengths to sample from, 5 will be enough. +wavelengths = np.linspace(450e-9, 700e-9, 5) + +mie_scatterer = dt.MieSphere( + position=(IMAGE_SIZE//4, IMAGE_SIZE//2, 2) * dt.units.pixel, #put particles in the middle + radius= 2e-6, + position_unit="pixel", + refractive_index=1.42, + L=10, +) + + +# Then, we define our darkfield microscope and image the particle through this. + + +optics = dt.Darkfield( + resolution=1e-6, + magnification=10, + wavelength=600e-9, + padding=(32, 32, 32, 32), + output_region=(0, 0, IMAGE_SIZE//2, IMAGE_SIZE), +) + +image = optics(mie_scatterer) + +# Sample the wavelengths. +imaged_particle_list = [] +for wavelength in wavelengths: + + # Create a darkfield microscope for a given wavelength. + single_wavelength_optics = dt.Darkfield( + resolution=1e-6, + magnification=10, + wavelength=wavelength, + padding=(32, 32, 32, 32), + output_region=(0, 0, IMAGE_SIZE, IMAGE_SIZE), + ) + + # Image the particle. + imaged_particle = single_wavelength_optics(mie_scatterer) + + # Add background noise. + imaged_particle = imaged_particle >> dt.Gaussian(0, 0.00015) + + # Append to list. + imaged_particle_list.append(imaged_particle) + +# Take the average of the images in the list. +darkfield_image = ( + sum(imaged_particle_list) / len(imaged_particle_list) +).resolve() +plt.imshow(darkfield_image, cmap="gray") +plt.show() \ No newline at end of file diff --git a/agent/deeptrack2-reference-to-code/examples/Fluorescence.py b/agent/deeptrack2-reference-to-code/examples/Fluorescence.py new file mode 100644 index 000000000..3d771b1d7 --- /dev/null +++ b/agent/deeptrack2-reference-to-code/examples/Fluorescence.py @@ -0,0 +1,36 @@ +import numpy as np +import deeptrack as dt +from matplotlib import pyplot as plt + +# ## Fluorescence +# Fluorescence images in `DeepTrack2` are simulated with discrete volumes to act as light sources (fluorophores) which emit light from a scatterer. +# +# Simulating fluorescence images are easy and straightforward to implement, we start by defining our microscope. +# Create a fluorescence microscope for a given wavelength. + +IMAGE_SIZE = 150 + +particle = dt.Sphere( + position=(IMAGE_SIZE//8, IMAGE_SIZE//4, 2) * dt.units.pixel, #put particles in the middle + radius= 0.3e-6, + position_unit="pixel", + refractive_index=1.42, +) + +fluorescence_optics = dt.Fluorescence( + NA=1.4, + resolution=1e-6, + magnification=10, + wavelength=600e-9, + padding=(32, 32, 32, 32), + output_region=(0, 0, IMAGE_SIZE//4, IMAGE_SIZE//2), +) + +# Image the particle. +imaged_particle = fluorescence_optics(particle) + +# Add background noise. +fluorescence_image = (imaged_particle >> dt.Gaussian(0, 0.001)).resolve() + +plt.imshow(fluorescence_image, cmap="gray") +plt.show() \ No newline at end of file diff --git a/agent/deeptrack2-reference-to-code/examples/Holography.py b/agent/deeptrack2-reference-to-code/examples/Holography.py new file mode 100644 index 000000000..5d4463758 --- /dev/null +++ b/agent/deeptrack2-reference-to-code/examples/Holography.py @@ -0,0 +1,59 @@ +import numpy as np +import deeptrack as dt +from matplotlib import pyplot as plt + + +IMAGE_SIZE = 150 + +# ## Holographic microscope +# The Holographic microscope returns an image with real and imaginary values, so we will need to define a helper function that converts these into floats so we can visualize the values with a plot. + + +def complex_to_float_f(): + """Converts a complex image to a float image, needed for Holography.""" + def inner(image): + image = image - 1 + output = np.zeros((*image.shape[:2], 2)) + output[..., 0:1] = np.real(image) + output[..., 1:2] = np.imag(image) + return output + return inner +complex_to_float = dt.Lambda(complex_to_float_f) + +# Define the holography microscope and resolve a particle. + +mie_scatterer = dt.MieSphere( + position=(IMAGE_SIZE//4, IMAGE_SIZE//2, 2) * dt.units.pixel, #put particles in the middle + radius= 2e-6, + position_unit="pixel", + refractive_index=1.42, + L=10, +) + +holography_microscope = dt.Holography( + resolution=1e-6, + magnification=10, + wavelength=600e-9, + padding=(32, 32, 32, 32), + output_region=(0, 0, IMAGE_SIZE//2, IMAGE_SIZE), + return_field=True, +) +# Get complex-valued image. +holography_image = holography_microscope(mie_scatterer).resolve() +print(f"Image shape = {holography_image.shape}") + +# Print a value in the image. +print(f"Pixel value of (0, 0) = {holography_image[0, 0, 0]}") + +# Convert to floats. +converted_image = complex_to_float(holography_image) +holography_real = converted_image[:, :, 0] +holography_imag = converted_image[:, :, 1] +holography_combined = holography_real + holography_imag + +fig, ax = plt.subplots(1, 3) +ax[0].imshow(holography_real, cmap="gray") +ax[1].imshow(holography_imag, cmap="gray") +ax[2].imshow(holography_combined, cmap="gray") + +plt.show() \ No newline at end of file diff --git a/agent/deeptrack2-reference-to-code/examples/ISCAT.py b/agent/deeptrack2-reference-to-code/examples/ISCAT.py new file mode 100644 index 000000000..91f7a44b6 --- /dev/null +++ b/agent/deeptrack2-reference-to-code/examples/ISCAT.py @@ -0,0 +1,27 @@ +import numpy as np +import deeptrack as dt +from matplotlib import pyplot as plt + + +IMAGE_SIZE = 150 + +mie_scatterer = dt.MieSphere( + position=(IMAGE_SIZE//2, IMAGE_SIZE//4, 2) * dt.units.pixel, #put particles in the middle + radius= 2e-6, + position_unit="pixel", + refractive_index=1.42, + L=10, +) + +iscat_microscope = dt.ISCAT( + resolution=1e-6, + magnification=10, + wavelength=600e-9, + padding=(32, 32, 32, 32), + output_region=(0, 0, IMAGE_SIZE, IMAGE_SIZE//2), +) + +iscat_image = iscat_microscope(mie_scatterer).resolve() + +plt.imshow(iscat_image, cmap="gray") +plt.show() \ No newline at end of file diff --git a/agent/deeptrack2-reference-to-code/examples/Lambda.py b/agent/deeptrack2-reference-to-code/examples/Lambda.py new file mode 100644 index 000000000..c716e525b --- /dev/null +++ b/agent/deeptrack2-reference-to-code/examples/Lambda.py @@ -0,0 +1,64 @@ +import deeptrack as dt +import numpy as np +from matplotlib import pyplot as plt + + +# Define the new operator fo channel average +def channel_average(axis=0): + def sny_function(image): + return np.mean(image, axis=axis) + return sny_function + +# Wraper it using dt.Lambad, make it possilbe to ba chained using `>>`, `^`, and `&` +syn_feature = dt.Lambda(function=channel_average) + +# Treat all input in one operation +syn_feature.__distributed__ = False + + +IMAGE_SIZE = 150 + +# Define 2 images channel under two wavelength +particle = dt.Sphere( + position= lambda: ( + np.array([ + 0.1*IMAGE_SIZE + 0.7*IMAGE_SIZE*np.random.random(), + 0.1*IMAGE_SIZE + 0.7*IMAGE_SIZE*np.random.random(), + ]) * dt.units.pixel + ), #put particles randomly + radius= 0.3e-6, + position_unit="pixel", + refractive_index=1.42, +) + + +fluorescence_optics_1 = dt.Fluorescence( + NA=1.4, + resolution=1e-6, + magnification=10, + wavelength=600e-9, + padding=(32, 32, 32, 32), + output_region=(0, 0, IMAGE_SIZE, IMAGE_SIZE), +) + + +fluorescence_optics_2 = dt.Fluorescence( + NA=1.4, + resolution=1e-6, + magnification=10, + wavelength=700e-9, + padding=(32, 32, 32, 32), + output_region=(0, 0, IMAGE_SIZE, IMAGE_SIZE), +) + +# Image the particle. +imaged_particle_1 = fluorescence_optics_1(particle) >> dt.NormalizeMinMax() +imaged_particle_2 = fluorescence_optics_2(particle) >> dt.NormalizeMinMax() + + +# Add background noise. +fluorescence_image = ((imaged_particle_1 & imaged_particle_2) + >> syn_feature >> dt.Gaussian(0, 0.001) >> dt.NormalizeMinMax()).update().resolve() + +plt.imshow(fluorescence_image) +plt.show() \ No newline at end of file diff --git a/agent/deeptrack2-reference-to-code/examples/To_PyTorch_Dataset.py b/agent/deeptrack2-reference-to-code/examples/To_PyTorch_Dataset.py new file mode 100644 index 000000000..c1aea1fc7 --- /dev/null +++ b/agent/deeptrack2-reference-to-code/examples/To_PyTorch_Dataset.py @@ -0,0 +1,85 @@ +import numpy as np +import deeptrack as dt +from matplotlib import pyplot as plt + +IMAGE_SIZE = 256 + +# Define the optical setup + +brightfield_microscope = dt.Brightfield( + NA=0.9, + resolution=1e-6, + magnification=10, + wavelength=680e-9, + output_region=(0, 0, IMAGE_SIZE, IMAGE_SIZE), + upscale=1, + padding=(32, 32, 32, 32), +) + +# Define the particle with random position + +particle = dt.MieSphere( + position=lambda: ( + np.array([ + 0.1*IMAGE_SIZE + 0.7*IMAGE_SIZE*np.random.random(), + 0.1*IMAGE_SIZE + 0.7*IMAGE_SIZE*np.random.random(), + ]) + ), + radius= lambda: (0.25 + 0.25*np.random.random())*1e-6, + intensity=10, +) + +imaged_particle_with_random_position = brightfield_microscope( + particle +) + +# Normalized Operation + +normalized_image_of_particle = ( + imaged_particle_with_random_position >> dt.NormalizeMinMax(0, 1) +) + +output_image = imaged_particle_with_random_position() + +# Visualization +plt.imshow(np.squeeze(output_image), cmap="gray") +plt.show() + +# Way1: Get Vector-based Ground-Truth +position_of_particle = particle.position() + +plt.imshow(np.squeeze(output_image), cmap="gray") +plt.scatter(position_of_particle[1], position_of_particle[0]) +plt.show() + +# Way2: Get Image-based Ground-Truth +CIRCLE_RADIUS = 3 + +def get_target_image(image, image_pipeline=normalized_image_of_particle): + """Create a binary image with the circles in the particle positions.""" + + target_image = np.zeros(image.shape) + x, y = np.meshgrid( + np.arange(0, image.shape[0]), + np.arange(0, image.shape[1]), + ) + + positions = dt.TakeProperties(image_pipeline, "position").resolve() + positions = np.reshape( + positions, (-1, 2) + ) # ensure (N, 2) shape, even for a single particle + + for position in positions: + distance_map = (x - position[1]) ** 2 + (y - position[0]) ** 2 + target_image[distance_map < CIRCLE_RADIUS**2] = 1 + + return target_image + +target_image = normalized_image_of_particle >> get_target_image + +# Visualization +plt.imshow(target_image(), cmap="gray") +plt.show() + +# Move Axis for Torch (Channel-first) +torch_data_pipeline = (normalized_image_of_particle & (normalized_image_of_particle >> get_target_image)) >> dt.MoveAxis(2, 0) \ No newline at end of file diff --git a/tutorials/2-examples/assets/local-LLM-envs/deeptrack_env/Dockerfile b/tutorials/2-examples/assets/local-LLM-envs/deeptrack_env/Dockerfile new file mode 100644 index 000000000..89d22902d --- /dev/null +++ b/tutorials/2-examples/assets/local-LLM-envs/deeptrack_env/Dockerfile @@ -0,0 +1,55 @@ +FROM continuumio/anaconda3 + +ENV DEBIAN_FRONTEND=noninteractive + + +RUN apt-get update && \ + apt-get install -y openssh-server sudo && \ + rm -rf /var/lib/apt/lists/* + + +RUN pip3 install --no-cache-dir torch torchvision \ + --index-url https://download.pytorch.org/whl/cu126 && \ + pip3 install --no-cache-dir deeptrack opencv-python && \ + conda install -y --quiet jupyterlab && \ + conda clean -afy + + +RUN useradd -m -s /bin/bash dev + + +RUN mkdir -p /home/dev/.ssh /run/sshd && \ + chmod 700 /home/dev/.ssh && \ + chown -R dev:dev /home/dev/.ssh + + +RUN sed -i \ + -e 's/^#PasswordAuthentication yes/PasswordAuthentication no/' \ + -e 's/^PasswordAuthentication yes/PasswordAuthentication no/' \ + -e 's/^#PubkeyAuthentication yes/PubkeyAuthentication yes/' \ + /etc/ssh/sshd_config + +RUN mkdir -p /root/.ssh && \ + echo "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAfNHQ94iIubf0eG9S+E6NayfPuzXcQPWriuBMTPhWnW xhujia@GU.GU.SE@softmatterlab01-p.gu.gu.se" \ + > /root/.ssh/authorized_keys && \ + chmod 700 /root/.ssh && \ + chmod 600 /root/.ssh/authorized_keys && \ + chown -R root:root /root/.ssh + +RUN echo 'source /opt/conda/etc/profile.d/conda.sh' >> /home/dev/.bashrc + +WORKDIR /a + +EXPOSE 22 8888 + +CMD ["/bin/bash", "-c", "\ + /usr/sbin/sshd && \ + jupyter lab \ + --notebook-dir=/ \ + --ip=0.0.0.0 \ + --port=8888 \ + --no-browser \ + --allow-root \ + --ServerApp.token='' \ + --ServerApp.password='' \ +"] \ No newline at end of file diff --git a/tutorials/2-examples/assets/local-LLM-envs/ollama/Dockerfile b/tutorials/2-examples/assets/local-LLM-envs/ollama/Dockerfile new file mode 100644 index 000000000..d87813337 --- /dev/null +++ b/tutorials/2-examples/assets/local-LLM-envs/ollama/Dockerfile @@ -0,0 +1,37 @@ +FROM ollama/ollama + +ENV DEBIAN_FRONTEND=noninteractive + +USER root + +RUN apt-get update \ + && apt-get install -y \ + build-essential \ + gcc g++ gfortran \ + cmake \ + make \ + git \ + wget \ + curl \ + python3 \ + python3-pip \ + python3-dev \ + python3-venv \ + python-is-python3 \ + vim \ + flex \ + hwloc \ + gawk \ + zlib1g-dev \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /opt + +RUN python3 -m venv .deeptrack && \ + . .deeptrack/bin/activate && \ + pip3 install --no-cache-dir torch torchvision \ + --index-url https://download.pytorch.org/whl/cu126 && \ + pip3 install --no-cache-dir deeptrack opencv-python && \ + echo 'source /opt/.deeptrack/bin/activate' >> /root/.bashrc + +CMD ["/bin/bash"] \ No newline at end of file