Skip to content

rslearn.models.dinov3

dinov3

DinoV3 model.

This code loads the DINOv3 model. You must obtain the model separately from Meta to use it. See https://github.com/facebookresearch/dinov3 for applicable license and copyright information.

DinoV3Models

Bases: StrEnum

Names for different DinoV3 images on torch hub.

Source code in rslearn/models/dinov3.py 
23
24
25
26
27
28
29
30
31
32
33
class DinoV3Models(StrEnum):
    """Names for different DinoV3 images on torch hub."""

    SMALL_WEB = "dinov3_vits16"
    SMALL_PLUS_WEB = "dinov3_vits16plus"
    BASE_WEB = "dinov3_vitb16"
    LARGE_WEB = "dinov3_vitl16"
    HUGE_PLUS_WEB = "dinov3_vith16plus"
    FULL_7B_WEB = "dinov3_vit7b16"
    LARGE_SATELLITE = "dinov3_vitl16_sat"
    FULL_7B_SATELLITE = "dinov3_vit7b16_sat"

DinoV3

Bases: FeatureExtractor

DinoV3 Backbones.

Must have the pretrained weights downloaded in checkpoint_dir for them to be loaded. See https://github.com/facebookresearch/dinov3?tab=readme-ov-file#pretrained-models

Only takes RGB as input. Expects normalized data (use the below normalizer).

Uses patch size 16. The input is resized to 256x256; when applying DinoV3 on segmentation or detection tasks with inputs larger than 256x256, it may be best to train and predict on 256x256 crops (using SplitConfig.patch_size argument).

Source code in rslearn/models/dinov3.py 
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
class DinoV3(FeatureExtractor):
    """DinoV3 Backbones.

    Must have the pretrained weights downloaded in checkpoint_dir for them to be loaded.
    See https://github.com/facebookresearch/dinov3?tab=readme-ov-file#pretrained-models

    Only takes RGB as input. Expects normalized data (use the below normalizer).

    Uses patch size 16. The input is resized to 256x256; when applying DinoV3 on
    segmentation or detection tasks with inputs larger than 256x256, it may be best to
    train and predict on 256x256 crops (using SplitConfig.patch_size argument).
    """

    image_size: int = 256
    patch_size: int = 16
    output_dim: int = 1024

    def _load_model(self, size: str, checkpoint_dir: str | None) -> torch.nn.Module:
        model_name = size.replace("_sat", "")
        if checkpoint_dir is not None:
            weights = str(Path(checkpoint_dir) / DINOV3_PTHS[size])
            return torch.hub.load(
                "facebookresearch/dinov3",
                model_name,
                weights=weights,
            )  # nosec
        return torch.hub.load("facebookresearch/dinov3", model_name, pretrained=False)  # nosec

    def __init__(
        self,
        checkpoint_dir: str | None,
        size: str = DinoV3Models.LARGE_SATELLITE,
        use_cls_token: bool = False,
        do_resizing: bool = True,
    ) -> None:
        """Instantiate a new DinoV3 instance.

        Args:
            checkpoint_dir: the local path to the pretrained weight dir. If None, we load the architecture
                only (randomly initialized).
            size: the model size, see class for various models.
            use_cls_token: use pooled class token (for classification), otherwise returns spatial feature map.
            do_resizing: whether to resize inputs to 256x256. Default true.
        """
        super().__init__()
        self.size = size
        self.checkpoint_dir = checkpoint_dir
        self.use_cls_token = use_cls_token
        self.do_resizing = do_resizing
        self.model = self._load_model(size, checkpoint_dir)

    def forward(self, context: ModelContext) -> FeatureMaps:
        """Forward pass for the dinov3 model.

        Args:
            context: the model context. Input dicts must include "image" key.

        Returns:
            a FeatureMaps with one feature map.
        """
        cur = torch.stack(
            [inp["image"].single_ts_to_chw_tensor() for inp in context.inputs],
            dim=0,
        )  # (B, C, H, W)

        if self.do_resizing and (
            cur.shape[2] != self.image_size or cur.shape[3] != self.image_size
        ):
            cur = torchvision.transforms.functional.resize(
                cur,
                [self.image_size, self.image_size],
            )

        if self.use_cls_token:
            features = self.model(cur)
        else:
            features = self.model.forward_features(cur)["x_norm_patchtokens"]
            batch_size, num_patches, _ = features.shape
            height, width = int(num_patches**0.5), int(num_patches**0.5)
            features = rearrange(features, "b (h w) d -> b d h w", h=height, w=width)

        return FeatureMaps([features])

    def get_backbone_channels(self) -> list:
        """Returns the output channels of this model when used as a backbone.

        The output channels is a list of (downsample_factor, depth) that corresponds
        to the feature maps that the backbone returns. For example, an element [2, 32]
        indicates that the corresponding feature map is 1/2 the input resolution and
        has 32 channels.
        """
        return [(self.patch_size, self.output_dim)]

forward 

forward(context: ModelContext) -> FeatureMaps

Forward pass for the dinov3 model.

Parameters:

Name Type Description Default
context ModelContext

the model context. Input dicts must include "image" key.

 required

Returns:

Type Description
FeatureMaps

a FeatureMaps with one feature map.
Source code in rslearn/models/dinov3.py 
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
def forward(self, context: ModelContext) -> FeatureMaps:
    """Forward pass for the dinov3 model.

    Args:
        context: the model context. Input dicts must include "image" key.

    Returns:
        a FeatureMaps with one feature map.
    """
    cur = torch.stack(
        [inp["image"].single_ts_to_chw_tensor() for inp in context.inputs],
        dim=0,
    )  # (B, C, H, W)

    if self.do_resizing and (
        cur.shape[2] != self.image_size or cur.shape[3] != self.image_size
    ):
        cur = torchvision.transforms.functional.resize(
            cur,
            [self.image_size, self.image_size],
        )

    if self.use_cls_token:
        features = self.model(cur)
    else:
        features = self.model.forward_features(cur)["x_norm_patchtokens"]
        batch_size, num_patches, _ = features.shape
        height, width = int(num_patches**0.5), int(num_patches**0.5)
        features = rearrange(features, "b (h w) d -> b d h w", h=height, w=width)

    return FeatureMaps([features])

get_backbone_channels 

get_backbone_channels() -> list

Returns the output channels of this model when used as a backbone.

The output channels is a list of (downsample_factor, depth) that corresponds to the feature maps that the backbone returns. For example, an element [2, 32] indicates that the corresponding feature map is 1/2 the input resolution and has 32 channels.

Source code in rslearn/models/dinov3.py 
129
130
131
132
133
134
135
136
137
def get_backbone_channels(self) -> list:
    """Returns the output channels of this model when used as a backbone.

    The output channels is a list of (downsample_factor, depth) that corresponds
    to the feature maps that the backbone returns. For example, an element [2, 32]
    indicates that the corresponding feature map is 1/2 the input resolution and
    has 32 channels.
    """
    return [(self.patch_size, self.output_dim)]

DinoV3Normalize

Bases: Transform

Normalize inputs using DinoV3 normalization.

Normalize "image" key in input according to Dino statistics from pretraining. Satellite pretraining has slightly different normalizing than the base image model so set 'satellite' depending on what pretrained model you are using.

Input "image" should be RGB-like image between 0-255.

Source code in rslearn/models/dinov3.py 
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
class DinoV3Normalize(Transform):
    """Normalize inputs using DinoV3 normalization.

    Normalize "image" key in input according to Dino statistics from pretraining. Satellite pretraining has slightly different normalizing than the base image model so set 'satellite' depending on what pretrained model you are using.

    Input "image" should be RGB-like image between 0-255.
    """

    def __init__(self, satellite: bool = True):
        """Initialize a new DinoV3Normalize."""
        super().__init__()
        self.satellite = satellite
        if satellite:
            mean = [0.430, 0.411, 0.296]
            std = [0.213, 0.156, 0.143]
        else:
            mean = [0.485, 0.456, 0.406]
            std = [0.229, 0.224, 0.225]

        self.normalize = Normalize(
            [value * 255 for value in mean],
            [value * 255 for value in std],
        )

    def forward(
        self, input_dict: dict[str, Any], target_dict: dict[str, Any]
    ) -> tuple[dict[str, Any], dict[str, Any]]:
        """Normalize the specified image with DinoV3 normalization.

        Args:
            input_dict: the input dictionary.
            target_dict: the target dictionary.

        Returns:
            normalized (input_dicts, target_dicts) tuple
        """
        return self.normalize(input_dict, target_dict)

forward 

forward(input_dict: dict[str, Any], target_dict: dict[str, Any]) -> tuple[dict[str, Any], dict[str, Any]]

Normalize the specified image with DinoV3 normalization.

Parameters:

Name Type Description Default
input_dict dict[str, Any]

the input dictionary.

 required
target_dict dict[str, Any]

the target dictionary.

 required

Returns:

Type Description
tuple[dict[str, Any], dict[str, Any]]

normalized (input_dicts, target_dicts) tuple
Source code in rslearn/models/dinov3.py 
164
165
166
167
168
169
170
171
172
173
174
175
176
def forward(
    self, input_dict: dict[str, Any], target_dict: dict[str, Any]
) -> tuple[dict[str, Any], dict[str, Any]]:
    """Normalize the specified image with DinoV3 normalization.

    Args:
        input_dict: the input dictionary.
        target_dict: the target dictionary.

    Returns:
        normalized (input_dicts, target_dicts) tuple
    """
    return self.normalize(input_dict, target_dict)