Skip to content

Dlt

Diffusers-style DLT layout generation package.

DLTConfig

Bases: ConfigMixin

Pipeline-level DLT configuration persisted with converted checkpoints.

Parameters:

Name Type Description Default
dataset_name str

Canonical dataset name.

'publaynet'
id2label dict[int | str, str] | None

Optional public label mapping. When omitted, shared dataset labels are used.

None
max_num_comp int | None

Maximum number of layout elements.

None
categories_num int | None

Internal category count including pad and mask/drop ids.

None
latent_dim int

Transformer latent dimension.

512
num_layers int

Number of transformer encoder layers.

4
num_heads int

Number of attention heads.

8
dropout_r float

Dropout probability.

0.0
activation str

Transformer activation.

'gelu'
cond_emb_size int

Box-condition embedding size.

224
cat_emb_size int

Category embedding size.

64
num_cont_timesteps int

Continuous DDPM training timesteps.

100
num_discrete_steps int

Discrete category diffusion steps.

10
beta_schedule str

DDPM beta schedule.

'squaredcos_cap_v2'
coordinate_range DLTCoordinateRange | str

Public coordinate range.

normalized_0_1
Source code in models/dlt/src/dlt/configuration_dlt.py
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
class DLTConfig(ConfigMixin):
    """Pipeline-level DLT configuration persisted with converted checkpoints.

    Args:
        dataset_name: Canonical dataset name.
        id2label: Optional public label mapping. When omitted, shared dataset
            labels are used.
        max_num_comp: Maximum number of layout elements.
        categories_num: Internal category count including pad and mask/drop ids.
        latent_dim: Transformer latent dimension.
        num_layers: Number of transformer encoder layers.
        num_heads: Number of attention heads.
        dropout_r: Dropout probability.
        activation: Transformer activation.
        cond_emb_size: Box-condition embedding size.
        cat_emb_size: Category embedding size.
        num_cont_timesteps: Continuous DDPM training timesteps.
        num_discrete_steps: Discrete category diffusion steps.
        beta_schedule: DDPM beta schedule.
        coordinate_range: Public coordinate range.
    """

    config_name = "dlt_config.json"

    @register_to_config
    def __init__(
        self,
        *,
        dataset_name: str = "publaynet",
        id2label: dict[int | str, str] | None = None,
        max_num_comp: int | None = None,
        categories_num: int | None = None,
        latent_dim: int = 512,
        num_layers: int = 4,
        num_heads: int = 8,
        dropout_r: float = 0.0,
        activation: str = "gelu",
        cond_emb_size: int = 224,
        cat_emb_size: int = 64,
        num_cont_timesteps: int = 100,
        num_discrete_steps: int = 10,
        beta_schedule: str = "squaredcos_cap_v2",
        coordinate_range: DLTCoordinateRange | str = DLTCoordinateRange.normalized_0_1,
    ) -> None:
        """Initialize DLT configuration."""
        dataset = normalize_dataset(dataset_name)
        labels = default_id2label(dataset) if id2label is None else id2label

        self.dataset_name = str(dataset)
        self.id2label = {int(key): value for key, value in labels.items()}
        self.max_num_comp = max_num_comp or max_elements_for_dataset(dataset)
        self.categories_num = categories_num or len(self.id2label) + 2

        self.latent_dim = latent_dim
        self.num_layers = num_layers
        self.num_heads = num_heads
        self.dropout_r = dropout_r
        self.activation = activation
        self.cond_emb_size = cond_emb_size
        self.cat_emb_size = cat_emb_size
        self.num_cont_timesteps = num_cont_timesteps
        self.num_discrete_steps = num_discrete_steps
        self.beta_schedule = beta_schedule
        self.coordinate_range = str(DLTCoordinateRange(coordinate_range))

    @property
    def label2id(self) -> dict[str, int]:
        """Return the public label-name to label-id mapping."""
        return {label: idx for idx, label in self.id2label.items()}

label2id property

label2id: dict[str, int]

Return the public label-name to label-id mapping.

__init__

__init__(
    *,
    dataset_name: str = "publaynet",
    id2label: dict[int | str, str] | None = None,
    max_num_comp: int | None = None,
    categories_num: int | None = None,
    latent_dim: int = 512,
    num_layers: int = 4,
    num_heads: int = 8,
    dropout_r: float = 0.0,
    activation: str = "gelu",
    cond_emb_size: int = 224,
    cat_emb_size: int = 64,
    num_cont_timesteps: int = 100,
    num_discrete_steps: int = 10,
    beta_schedule: str = "squaredcos_cap_v2",
    coordinate_range: DLTCoordinateRange
    | str = DLTCoordinateRange.normalized_0_1,
) -> None

Initialize DLT configuration.

Source code in models/dlt/src/dlt/configuration_dlt.py
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
@register_to_config
def __init__(
    self,
    *,
    dataset_name: str = "publaynet",
    id2label: dict[int | str, str] | None = None,
    max_num_comp: int | None = None,
    categories_num: int | None = None,
    latent_dim: int = 512,
    num_layers: int = 4,
    num_heads: int = 8,
    dropout_r: float = 0.0,
    activation: str = "gelu",
    cond_emb_size: int = 224,
    cat_emb_size: int = 64,
    num_cont_timesteps: int = 100,
    num_discrete_steps: int = 10,
    beta_schedule: str = "squaredcos_cap_v2",
    coordinate_range: DLTCoordinateRange | str = DLTCoordinateRange.normalized_0_1,
) -> None:
    """Initialize DLT configuration."""
    dataset = normalize_dataset(dataset_name)
    labels = default_id2label(dataset) if id2label is None else id2label

    self.dataset_name = str(dataset)
    self.id2label = {int(key): value for key, value in labels.items()}
    self.max_num_comp = max_num_comp or max_elements_for_dataset(dataset)
    self.categories_num = categories_num or len(self.id2label) + 2

    self.latent_dim = latent_dim
    self.num_layers = num_layers
    self.num_heads = num_heads
    self.dropout_r = dropout_r
    self.activation = activation
    self.cond_emb_size = cond_emb_size
    self.cat_emb_size = cat_emb_size
    self.num_cont_timesteps = num_cont_timesteps
    self.num_discrete_steps = num_discrete_steps
    self.beta_schedule = beta_schedule
    self.coordinate_range = str(DLTCoordinateRange(coordinate_range))

DLT

Bases: ModelMixin, ConfigMixin

Joint continuous/discrete DLT denoiser.

The module names intentionally match released checkpoint keys so model.save_pretrained directories can load without key rewriting.

Parameters:

Name Type Description Default
categories_num int

Internal category count including pad and mask/drop ids.

required
latent_dim int

Transformer latent dimension.

256
num_layers int

Number of transformer encoder layers.

4
num_heads int

Number of attention heads.

4
dropout_r float

Dropout probability.

0.0
activation str

Transformer activation.

'gelu'
cond_emb_size int

Box-condition embedding size.

224
cat_emb_size int

Category embedding size.

64
Source code in models/dlt/src/dlt/modeling_dlt.py
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
class DLT(ModelMixin, ConfigMixin):
    """Joint continuous/discrete DLT denoiser.

    The module names intentionally match released checkpoint keys so
    ``model.save_pretrained`` directories can load without key rewriting.

    Args:
        categories_num: Internal category count including pad and mask/drop ids.
        latent_dim: Transformer latent dimension.
        num_layers: Number of transformer encoder layers.
        num_heads: Number of attention heads.
        dropout_r: Dropout probability.
        activation: Transformer activation.
        cond_emb_size: Box-condition embedding size.
        cat_emb_size: Category embedding size.
    """

    config_name = "model_config.json"

    @register_to_config
    def __init__(
        self,
        categories_num: int,
        latent_dim: int = 256,
        num_layers: int = 4,
        num_heads: int = 4,
        dropout_r: float = 0.0,
        activation: str = "gelu",
        cond_emb_size: int = 224,
        cat_emb_size: int = 64,
    ) -> None:
        """Initialize the DLT denoiser."""
        super().__init__()
        self.latent_dim = latent_dim
        self.dropout_r = dropout_r
        self.categories_num = categories_num
        self.seq_pos_enc = PositionalEncoding(self.latent_dim, self.dropout_r)
        self.cat_emb = nn.Parameter(torch.randn(self.categories_num, cat_emb_size))
        self.cond_mask_box_emb = nn.Parameter(torch.randn(2, cond_emb_size))
        self.cond_mask_cat_emb = nn.Parameter(torch.randn(2, cat_emb_size))

        seq_trans_encoder_layer = nn.TransformerEncoderLayer(
            d_model=self.latent_dim,
            nhead=num_heads,
            dim_feedforward=self.latent_dim * 2,
            dropout=dropout_r,
            activation=activation,
        )
        self.seqTransEncoder = nn.TransformerEncoder(
            seq_trans_encoder_layer, num_layers=num_layers
        )
        self.embed_timestep = TimestepEmbedder(self.latent_dim, self.seq_pos_enc)
        self.output_process = nn.Sequential(nn.Linear(self.latent_dim, 4))
        self.output_cls = nn.Sequential(nn.Linear(self.latent_dim, categories_num))
        self.size_emb = nn.Sequential(nn.Linear(2, cond_emb_size))
        self.loc_emb = nn.Sequential(nn.Linear(2, cond_emb_size))

    def forward(
        self,
        sample: dict[
            str,
            Float[torch.Tensor, "batch elements channels"]
            | Int[torch.Tensor, "batch elements"],
        ],
        noisy_sample: dict[
            str,
            Float[torch.Tensor, "batch elements channels"]
            | Int[torch.Tensor, "batch elements"],
        ],
        timesteps: Int[torch.Tensor, "batch"],
        return_dict: bool = False,
    ) -> (
        DLTModelOutput
        | tuple[
            Float[torch.Tensor, "batch elements 4"],
            Float[torch.Tensor, "batch elements categories"],
        ]
    ):
        """Predict clean boxes and category logits for a noisy layout.

        Args:
            sample: DLT-format conditioning batch with ``box_cond``,
                ``cat``, ``mask_box``, and ``mask_cat``.
            noisy_sample: Current noisy ``box`` and ``cat`` tensors.
            timesteps: Continuous diffusion timestep per batch item.
            return_dict: Whether to return ``DLTModelOutput``.

        Returns:
            Either a two-tuple ``(box, logits)`` for checkpoint-compatible
            callers or a dataclass output.
        """
        cat_input = (
            noisy_sample["cat"] * sample["mask_cat"]
            + (1 - sample["mask_cat"]) * sample["cat"]
        )
        cat_input_flat = rearrange(cat_input, "b c -> (b c)")
        sample_tensor = (
            sample["mask_box"] * noisy_sample["box"]
            + (1 - sample["mask_box"]) * sample["box_cond"]
        )

        xy = sample_tensor[:, :, :2]
        wh = sample_tensor[:, :, 2:]

        elem_cat_emb = self.cat_emb[cat_input_flat, :]
        elem_cat_emb = rearrange(
            elem_cat_emb, "(b c) d -> b c d", b=noisy_sample["box"].shape[0]
        )

        def mask_to_emb(
            mask: Int[torch.Tensor, "batch elements"],
            cond_mask_emb: Float[torch.Tensor, "mask channels"],
        ) -> Float[torch.Tensor, "batch elements channels"]:
            mask_flat = rearrange(mask, "b c -> (b c)").long()
            mask_all_emb = cond_mask_emb[mask_flat, :]
            return rearrange(mask_all_emb, "(b c) d -> b c d", b=mask.shape[0])

        emb_mask_wh = mask_to_emb(sample["mask_box"][:, :, 2], self.cond_mask_box_emb)
        emb_mask_xy = mask_to_emb(sample["mask_box"][:, :, 0], self.cond_mask_box_emb)
        emb_mask_cl = mask_to_emb(sample["mask_cat"], self.cond_mask_cat_emb)
        t_emb = self.embed_timestep(timesteps)

        size_emb = self.size_emb(wh) + emb_mask_wh
        loc_emb = self.loc_emb(xy) + emb_mask_xy
        elem_cat_emb = elem_cat_emb + emb_mask_cl

        tokens_emb = torch.cat([size_emb, loc_emb, elem_cat_emb], dim=-1)
        tokens_emb = rearrange(tokens_emb, "b c d -> c b d")
        xseq = torch.cat((t_emb, tokens_emb), dim=0)
        xseq = self.seq_pos_enc(xseq)

        output = self.seqTransEncoder(xseq)[1:]
        output = rearrange(output, "c b d -> b c d")
        output_box = self.output_process(output)
        output_cls = self.output_cls(output)
        if not return_dict:
            return output_box, output_cls
        return DLTModelOutput(box=output_box, logits=output_cls)

    def save_pretrained(
        self,
        save_directory: str | os.PathLike[str],
        is_main_process: bool = True,
        save_function: Callable[..., None] | None = None,
        safe_serialization: bool = False,
        variant: str | None = None,
        max_shard_size: int | str = "10GB",
        push_to_hub: bool = False,
        use_flashpack: bool = False,
        **kwargs: str | int | bool | float | None,
    ) -> None:
        """Save the model with PyTorch serialization by default.

        DLT keeps shared positional-encoding buffers that safetensors refuses
        to flatten.
        """
        super().save_pretrained(
            save_directory,
            is_main_process=is_main_process,
            save_function=save_function,
            safe_serialization=safe_serialization,
            variant=variant,
            max_shard_size=max_shard_size,
            push_to_hub=push_to_hub,
            use_flashpack=use_flashpack,
            **kwargs,
        )

__init__

__init__(
    categories_num: int,
    latent_dim: int = 256,
    num_layers: int = 4,
    num_heads: int = 4,
    dropout_r: float = 0.0,
    activation: str = "gelu",
    cond_emb_size: int = 224,
    cat_emb_size: int = 64,
) -> None

Initialize the DLT denoiser.

Source code in models/dlt/src/dlt/modeling_dlt.py
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
@register_to_config
def __init__(
    self,
    categories_num: int,
    latent_dim: int = 256,
    num_layers: int = 4,
    num_heads: int = 4,
    dropout_r: float = 0.0,
    activation: str = "gelu",
    cond_emb_size: int = 224,
    cat_emb_size: int = 64,
) -> None:
    """Initialize the DLT denoiser."""
    super().__init__()
    self.latent_dim = latent_dim
    self.dropout_r = dropout_r
    self.categories_num = categories_num
    self.seq_pos_enc = PositionalEncoding(self.latent_dim, self.dropout_r)
    self.cat_emb = nn.Parameter(torch.randn(self.categories_num, cat_emb_size))
    self.cond_mask_box_emb = nn.Parameter(torch.randn(2, cond_emb_size))
    self.cond_mask_cat_emb = nn.Parameter(torch.randn(2, cat_emb_size))

    seq_trans_encoder_layer = nn.TransformerEncoderLayer(
        d_model=self.latent_dim,
        nhead=num_heads,
        dim_feedforward=self.latent_dim * 2,
        dropout=dropout_r,
        activation=activation,
    )
    self.seqTransEncoder = nn.TransformerEncoder(
        seq_trans_encoder_layer, num_layers=num_layers
    )
    self.embed_timestep = TimestepEmbedder(self.latent_dim, self.seq_pos_enc)
    self.output_process = nn.Sequential(nn.Linear(self.latent_dim, 4))
    self.output_cls = nn.Sequential(nn.Linear(self.latent_dim, categories_num))
    self.size_emb = nn.Sequential(nn.Linear(2, cond_emb_size))
    self.loc_emb = nn.Sequential(nn.Linear(2, cond_emb_size))

forward

forward(
    sample: dict[
        str,
        Float[Tensor, "batch elements channels"]
        | Int[Tensor, "batch elements"],
    ],
    noisy_sample: dict[
        str,
        Float[Tensor, "batch elements channels"]
        | Int[Tensor, "batch elements"],
    ],
    timesteps: Int[Tensor, "batch"],
    return_dict: bool = False,
) -> (
    DLTModelOutput
    | tuple[
        Float[torch.Tensor, "batch elements 4"],
        Float[torch.Tensor, "batch elements categories"],
    ]
)

Predict clean boxes and category logits for a noisy layout.

Parameters:

Name Type Description Default
sample dict[str, Float[Tensor, 'batch elements channels'] | Int[Tensor, 'batch elements']]

DLT-format conditioning batch with box_cond, cat, mask_box, and mask_cat.

required
noisy_sample dict[str, Float[Tensor, 'batch elements channels'] | Int[Tensor, 'batch elements']]

Current noisy box and cat tensors.

required
timesteps Int[Tensor, 'batch']

Continuous diffusion timestep per batch item.

required
return_dict bool

Whether to return DLTModelOutput.

False

Returns:

Type Description
DLTModelOutput | tuple[Float[Tensor, 'batch elements 4'], Float[Tensor, 'batch elements categories']]

Either a two-tuple (box, logits) for checkpoint-compatible

DLTModelOutput | tuple[Float[Tensor, 'batch elements 4'], Float[Tensor, 'batch elements categories']]

callers or a dataclass output.

Source code in models/dlt/src/dlt/modeling_dlt.py
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
def forward(
    self,
    sample: dict[
        str,
        Float[torch.Tensor, "batch elements channels"]
        | Int[torch.Tensor, "batch elements"],
    ],
    noisy_sample: dict[
        str,
        Float[torch.Tensor, "batch elements channels"]
        | Int[torch.Tensor, "batch elements"],
    ],
    timesteps: Int[torch.Tensor, "batch"],
    return_dict: bool = False,
) -> (
    DLTModelOutput
    | tuple[
        Float[torch.Tensor, "batch elements 4"],
        Float[torch.Tensor, "batch elements categories"],
    ]
):
    """Predict clean boxes and category logits for a noisy layout.

    Args:
        sample: DLT-format conditioning batch with ``box_cond``,
            ``cat``, ``mask_box``, and ``mask_cat``.
        noisy_sample: Current noisy ``box`` and ``cat`` tensors.
        timesteps: Continuous diffusion timestep per batch item.
        return_dict: Whether to return ``DLTModelOutput``.

    Returns:
        Either a two-tuple ``(box, logits)`` for checkpoint-compatible
        callers or a dataclass output.
    """
    cat_input = (
        noisy_sample["cat"] * sample["mask_cat"]
        + (1 - sample["mask_cat"]) * sample["cat"]
    )
    cat_input_flat = rearrange(cat_input, "b c -> (b c)")
    sample_tensor = (
        sample["mask_box"] * noisy_sample["box"]
        + (1 - sample["mask_box"]) * sample["box_cond"]
    )

    xy = sample_tensor[:, :, :2]
    wh = sample_tensor[:, :, 2:]

    elem_cat_emb = self.cat_emb[cat_input_flat, :]
    elem_cat_emb = rearrange(
        elem_cat_emb, "(b c) d -> b c d", b=noisy_sample["box"].shape[0]
    )

    def mask_to_emb(
        mask: Int[torch.Tensor, "batch elements"],
        cond_mask_emb: Float[torch.Tensor, "mask channels"],
    ) -> Float[torch.Tensor, "batch elements channels"]:
        mask_flat = rearrange(mask, "b c -> (b c)").long()
        mask_all_emb = cond_mask_emb[mask_flat, :]
        return rearrange(mask_all_emb, "(b c) d -> b c d", b=mask.shape[0])

    emb_mask_wh = mask_to_emb(sample["mask_box"][:, :, 2], self.cond_mask_box_emb)
    emb_mask_xy = mask_to_emb(sample["mask_box"][:, :, 0], self.cond_mask_box_emb)
    emb_mask_cl = mask_to_emb(sample["mask_cat"], self.cond_mask_cat_emb)
    t_emb = self.embed_timestep(timesteps)

    size_emb = self.size_emb(wh) + emb_mask_wh
    loc_emb = self.loc_emb(xy) + emb_mask_xy
    elem_cat_emb = elem_cat_emb + emb_mask_cl

    tokens_emb = torch.cat([size_emb, loc_emb, elem_cat_emb], dim=-1)
    tokens_emb = rearrange(tokens_emb, "b c d -> c b d")
    xseq = torch.cat((t_emb, tokens_emb), dim=0)
    xseq = self.seq_pos_enc(xseq)

    output = self.seqTransEncoder(xseq)[1:]
    output = rearrange(output, "c b d -> b c d")
    output_box = self.output_process(output)
    output_cls = self.output_cls(output)
    if not return_dict:
        return output_box, output_cls
    return DLTModelOutput(box=output_box, logits=output_cls)

save_pretrained

save_pretrained(
    save_directory: str | PathLike[str],
    is_main_process: bool = True,
    save_function: Callable[..., None] | None = None,
    safe_serialization: bool = False,
    variant: str | None = None,
    max_shard_size: int | str = "10GB",
    push_to_hub: bool = False,
    use_flashpack: bool = False,
    **kwargs: str | int | bool | float | None,
) -> None

Save the model with PyTorch serialization by default.

DLT keeps shared positional-encoding buffers that safetensors refuses to flatten.

Source code in models/dlt/src/dlt/modeling_dlt.py
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
def save_pretrained(
    self,
    save_directory: str | os.PathLike[str],
    is_main_process: bool = True,
    save_function: Callable[..., None] | None = None,
    safe_serialization: bool = False,
    variant: str | None = None,
    max_shard_size: int | str = "10GB",
    push_to_hub: bool = False,
    use_flashpack: bool = False,
    **kwargs: str | int | bool | float | None,
) -> None:
    """Save the model with PyTorch serialization by default.

    DLT keeps shared positional-encoding buffers that safetensors refuses
    to flatten.
    """
    super().save_pretrained(
        save_directory,
        is_main_process=is_main_process,
        save_function=save_function,
        safe_serialization=safe_serialization,
        variant=variant,
        max_shard_size=max_shard_size,
        push_to_hub=push_to_hub,
        use_flashpack=use_flashpack,
        **kwargs,
    )

DLTModelOutput dataclass

Bases: BaseOutput

Output returned by the DLT denoiser.

Attributes:

Name Type Description
box Float[Tensor, 'batch elements 4']

Predicted clean internal-range boxes.

logits Float[Tensor, 'batch elements categories']

Category logits.

Source code in models/dlt/src/dlt/modeling_dlt.py
19
20
21
22
23
24
25
26
27
28
29
@dataclass
class DLTModelOutput(BaseOutput):
    """Output returned by the DLT denoiser.

    Attributes:
        box: Predicted clean internal-range boxes.
        logits: Category logits.
    """

    box: Float[torch.Tensor, "batch elements 4"]
    logits: Float[torch.Tensor, "batch elements categories"]

DLTConditionAlias

Bases: StrEnum

DLT checkpoint condition aliases.

Source code in models/dlt/src/dlt/pipeline_dlt.py
28
29
30
31
32
33
class DLTConditionAlias(StrEnum):
    """DLT checkpoint condition aliases."""

    all = auto()
    whole_box = auto()
    loc = auto()

DLTPipeline

Bases: DiffusionPipeline

Generate layouts with a converted DLT checkpoint.

Parameters:

Name Type Description Default
model DLT

DLT denoiser.

required
scheduler DLTJointDiffusionScheduler

Joint box/category scheduler.

required
config DLTConfig

Pipeline configuration.

required
processor DLTProcessor | None

Layout processor.

None
Source code in models/dlt/src/dlt/pipeline_dlt.py
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
class DLTPipeline(DiffusionPipeline):
    """Generate layouts with a converted DLT checkpoint.

    Args:
        model: DLT denoiser.
        scheduler: Joint box/category scheduler.
        config: Pipeline configuration.
        processor: Layout processor.
    """

    model_cpu_offload_seq = "model"
    _optional_components = ["processor"]

    def __init__(
        self,
        model: DLT,
        scheduler: DLTJointDiffusionScheduler,
        config: DLTConfig,
        processor: DLTProcessor | None = None,
    ) -> None:
        """Initialize a DLT pipeline."""
        super().__init__()
        self.register_modules(model=model, scheduler=scheduler)
        self.dlt_config = config
        self.processor = processor or DLTProcessor(
            dataset=self.dlt_config.dataset_name,
            labels=tuple(self.dlt_config.id2label.values()),
            max_num_comp=self.dlt_config.max_num_comp,
        )
        self.model.eval()

    @property
    def components(self) -> PipelineComponents:
        """Return serializable pipeline components."""
        return {
            "model": self.model,
            "scheduler": self.scheduler,
            "processor": self.processor,
        }

    @torch.no_grad()
    def __call__(
        self,
        *,
        batch_size: int = 1,
        seed: int | None = None,
        generator: torch.Generator | None = None,
        condition_type: ConditionType | str | None = ConditionType.unconditional,
        labels: Int[torch.Tensor, "batch elements"] | None = None,
        bbox: Float[torch.Tensor, "batch elements 4"] | None = None,
        mask: Bool[torch.Tensor, "batch elements"] | None = None,
        num_elements: int | list[int] | Int[torch.Tensor, "batch"] | None = None,
        box_format: BoxFormat | str = BoxFormat.xywh,
        normalized: bool = True,
        canvas_size: tuple[int, int] | None = None,
        num_inference_steps: int | None = None,
        temperature: float | None = None,
        output_type: OutputType | str = OutputType.dataclass,
        return_intermediates: bool = False,
    ) -> (
        LayoutGenerationOutput
        | dict[
            str,
            Float[torch.Tensor, "..."]
            | Int[torch.Tensor, "..."]
            | Bool[torch.Tensor, "..."]
            | dict[int, str]
            | list[Float[torch.Tensor, "..."]]
            | dict[str, str]
            | None,
        ]
    ):
        """Run DLT joint denoising and return generated layouts.

        Args:
            batch_size: Number of layouts to generate.
            seed: Optional seed used when ``generator`` is absent.
            generator: Optional torch generator. Takes precedence over ``seed``.
            condition_type: Canonical condition or DLT checkpoint alias.
            labels: Optional public labels for conditioned modes.
            bbox: Optional public boxes for conditioned modes.
            mask: Optional valid-element mask.
            num_elements: Optional valid element count for unconditional calls.
            box_format: Input box format.
            normalized: Whether input boxes are normalized.
            canvas_size: Pixel canvas size for non-normalized boxes.
            num_inference_steps: Number of reverse diffusion steps.
            temperature: Optional category sampling temperature override.
            output_type: ``"dataclass"`` or ``"dict"``.
            return_intermediates: Whether to include denoising trajectory.

        Returns:
            Layout generation output dataclass or dictionary.

        Raises:
            ValueError: If the condition or output type is unsupported.
        """
        canonical = normalize_condition_type(condition_type)
        output_kind = OutputType(output_type)
        if generator is None and seed is not None:
            generator = torch.Generator(device=self.device).manual_seed(seed)
        if canonical is ConditionType.unconditional:
            processed = self.processor.empty_condition(
                batch_size=batch_size, device=self.device
            )
            if num_elements is not None:
                lengths = torch.as_tensor(
                    num_elements, dtype=torch.long, device=self.device
                )
                if lengths.ndim == 0:
                    lengths = lengths.repeat(batch_size)
                processed["mask"] = (
                    torch.arange(self.processor.max_num_comp, device=self.device)[
                        None, :
                    ]
                    < lengths[:, None]
                )
        else:
            _require_condition_inputs(
                condition_type=condition_type, bbox=bbox, labels=labels
            )
            processed = self.processor(
                bbox=bbox,
                labels=labels,
                mask=mask,
                box_format=box_format,
                normalized=normalized,
                canvas_size=canvas_size,
                device=self.device,
            )
            batch_size = processed["box"].shape[0]
        mask_box, mask_cat = self.processor.condition_masks(
            str(canonical), mask=processed["mask"]
        )
        sample = {
            "box": processed["box"],
            "box_cond": processed["box_cond"],
            "cat": processed["cat"],
            "mask_box": mask_box,
            "mask_cat": mask_cat,
        }
        noisy_batch = {
            "box": torch.randn(
                processed["box"].shape,
                dtype=processed["box"].dtype,
                device=self.device,
                generator=generator,
            ),
            "cat": torch.full(
                processed["cat"].shape,
                self.processor.mask_category_id,
                dtype=torch.long,
                device=self.device,
            ),
        }
        original_temperature = self.scheduler.temperature
        if temperature is not None:
            self.scheduler.temperature = temperature
        steps = max(1, num_inference_steps or self.scheduler.num_train_timesteps)
        trajectory: list[Float[torch.Tensor, "batch elements 4"]] | None = (
            [] if return_intermediates else None
        )
        bbox_step: DLTJointSchedulerOutput | None = None
        try:
            for i in range(steps - 1, -1, -1):
                t_value = min(i, self.scheduler.num_train_timesteps - 1)
                t = torch.tensor([t_value] * batch_size, device=self.device)
                bbox_pred, cat_pred = self.model(sample, noisy_batch, timesteps=t)
                bbox_step, cat_step = self.scheduler.step_jointly(
                    bbox_pred,
                    {"cat": cat_pred},
                    timestep=t,
                    sample=noisy_batch["box"],
                    generator=generator,
                )
                noisy_batch["box"] = bbox_step.prev_sample
                noisy_batch["cat"] = cat_step["cat"]
                if trajectory is not None:
                    trajectory.append(noisy_batch["box"].detach().cpu())
        finally:
            self.scheduler.temperature = original_temperature
        if bbox_step is None:
            raise RuntimeError("DLT denoising did not run any scheduler steps")

        final_box = (
            sample["mask_box"] * bbox_step.pred_original_sample
            + (1 - sample["mask_box"]) * sample["box_cond"]
        )
        final_cat = (
            sample["mask_cat"] * noisy_batch["cat"]
            + (1 - sample["mask_cat"]) * sample["cat"]
        )
        valid_mask = processed["mask"].detach().cpu()
        output = LayoutGenerationOutput(
            bbox=self.processor.internal_to_public_boxes(final_box).detach().cpu()
            * valid_mask.unsqueeze(-1),
            labels=self.processor.internal_to_public_labels(
                final_cat, processed["mask"]
            )
            .detach()
            .cpu(),
            mask=valid_mask,
            id2label=self.processor.id2label,
            trajectory=trajectory,
            intermediates={"condition_type": str(canonical)}
            if return_intermediates
            else None,
        )
        return _format_pipeline_output(output, output_kind)

    generate = __call__

    def save_pretrained(self, save_directory: str | Path) -> None:
        """Persist DLT model, scheduler, and pipeline metadata."""
        super().save_pretrained(save_directory, safe_serialization=False)
        self.dlt_config.save_config(save_directory)

    @classmethod
    def from_pretrained(cls, pretrained_model_name_or_path: str | Path) -> Self:
        """Load a saved DLT pipeline."""
        config_dict, _ = DLTConfig.load_config(
            pretrained_model_name_or_path, return_unused_kwargs=True
        )
        config = cast(DLTConfig, DLTConfig.from_config(config_dict))
        pipe = super().from_pretrained(pretrained_model_name_or_path, config=config)
        pipe.dlt_config = config
        pipe.processor = DLTProcessor(
            dataset=config.dataset_name,
            labels=tuple(config.id2label.values()),
            max_num_comp=config.max_num_comp,
        )
        return pipe

components property

components: PipelineComponents

Return serializable pipeline components.

__init__

__init__(
    model: DLT,
    scheduler: DLTJointDiffusionScheduler,
    config: DLTConfig,
    processor: DLTProcessor | None = None,
) -> None

Initialize a DLT pipeline.

Source code in models/dlt/src/dlt/pipeline_dlt.py
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
def __init__(
    self,
    model: DLT,
    scheduler: DLTJointDiffusionScheduler,
    config: DLTConfig,
    processor: DLTProcessor | None = None,
) -> None:
    """Initialize a DLT pipeline."""
    super().__init__()
    self.register_modules(model=model, scheduler=scheduler)
    self.dlt_config = config
    self.processor = processor or DLTProcessor(
        dataset=self.dlt_config.dataset_name,
        labels=tuple(self.dlt_config.id2label.values()),
        max_num_comp=self.dlt_config.max_num_comp,
    )
    self.model.eval()

__call__

__call__(
    *,
    batch_size: int = 1,
    seed: int | None = None,
    generator: Generator | None = None,
    condition_type: ConditionType
    | str
    | None = ConditionType.unconditional,
    labels: Int[Tensor, "batch elements"] | None = None,
    bbox: Float[Tensor, "batch elements 4"] | None = None,
    mask: Bool[Tensor, "batch elements"] | None = None,
    num_elements: int
    | list[int]
    | Int[Tensor, batch]
    | None = None,
    box_format: BoxFormat | str = BoxFormat.xywh,
    normalized: bool = True,
    canvas_size: tuple[int, int] | None = None,
    num_inference_steps: int | None = None,
    temperature: float | None = None,
    output_type: OutputType | str = OutputType.dataclass,
    return_intermediates: bool = False,
) -> (
    LayoutGenerationOutput
    | dict[
        str,
        Float[torch.Tensor, ...]
        | Int[torch.Tensor, ...]
        | Bool[torch.Tensor, ...]
        | dict[int, str]
        | list[Float[torch.Tensor, ...]]
        | dict[str, str]
        | None,
    ]
)

Run DLT joint denoising and return generated layouts.

Parameters:

Name Type Description Default
batch_size int

Number of layouts to generate.

1
seed int | None

Optional seed used when generator is absent.

None
generator Generator | None

Optional torch generator. Takes precedence over seed.

None
condition_type ConditionType | str | None

Canonical condition or DLT checkpoint alias.

unconditional
labels Int[Tensor, 'batch elements'] | None

Optional public labels for conditioned modes.

None
bbox Float[Tensor, 'batch elements 4'] | None

Optional public boxes for conditioned modes.

None
mask Bool[Tensor, 'batch elements'] | None

Optional valid-element mask.

None
num_elements int | list[int] | Int[Tensor, batch] | None

Optional valid element count for unconditional calls.

None
box_format BoxFormat | str

Input box format.

xywh
normalized bool

Whether input boxes are normalized.

True
canvas_size tuple[int, int] | None

Pixel canvas size for non-normalized boxes.

None
num_inference_steps int | None

Number of reverse diffusion steps.

None
temperature float | None

Optional category sampling temperature override.

None
output_type OutputType | str

"dataclass" or "dict".

dataclass
return_intermediates bool

Whether to include denoising trajectory.

False

Returns:

Type Description
LayoutGenerationOutput | dict[str, Float[Tensor, ...] | Int[Tensor, ...] | Bool[Tensor, ...] | dict[int, str] | list[Float[Tensor, ...]] | dict[str, str] | None]

Layout generation output dataclass or dictionary.

Raises:

Type Description
ValueError

If the condition or output type is unsupported.

Source code in models/dlt/src/dlt/pipeline_dlt.py
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
@torch.no_grad()
def __call__(
    self,
    *,
    batch_size: int = 1,
    seed: int | None = None,
    generator: torch.Generator | None = None,
    condition_type: ConditionType | str | None = ConditionType.unconditional,
    labels: Int[torch.Tensor, "batch elements"] | None = None,
    bbox: Float[torch.Tensor, "batch elements 4"] | None = None,
    mask: Bool[torch.Tensor, "batch elements"] | None = None,
    num_elements: int | list[int] | Int[torch.Tensor, "batch"] | None = None,
    box_format: BoxFormat | str = BoxFormat.xywh,
    normalized: bool = True,
    canvas_size: tuple[int, int] | None = None,
    num_inference_steps: int | None = None,
    temperature: float | None = None,
    output_type: OutputType | str = OutputType.dataclass,
    return_intermediates: bool = False,
) -> (
    LayoutGenerationOutput
    | dict[
        str,
        Float[torch.Tensor, "..."]
        | Int[torch.Tensor, "..."]
        | Bool[torch.Tensor, "..."]
        | dict[int, str]
        | list[Float[torch.Tensor, "..."]]
        | dict[str, str]
        | None,
    ]
):
    """Run DLT joint denoising and return generated layouts.

    Args:
        batch_size: Number of layouts to generate.
        seed: Optional seed used when ``generator`` is absent.
        generator: Optional torch generator. Takes precedence over ``seed``.
        condition_type: Canonical condition or DLT checkpoint alias.
        labels: Optional public labels for conditioned modes.
        bbox: Optional public boxes for conditioned modes.
        mask: Optional valid-element mask.
        num_elements: Optional valid element count for unconditional calls.
        box_format: Input box format.
        normalized: Whether input boxes are normalized.
        canvas_size: Pixel canvas size for non-normalized boxes.
        num_inference_steps: Number of reverse diffusion steps.
        temperature: Optional category sampling temperature override.
        output_type: ``"dataclass"`` or ``"dict"``.
        return_intermediates: Whether to include denoising trajectory.

    Returns:
        Layout generation output dataclass or dictionary.

    Raises:
        ValueError: If the condition or output type is unsupported.
    """
    canonical = normalize_condition_type(condition_type)
    output_kind = OutputType(output_type)
    if generator is None and seed is not None:
        generator = torch.Generator(device=self.device).manual_seed(seed)
    if canonical is ConditionType.unconditional:
        processed = self.processor.empty_condition(
            batch_size=batch_size, device=self.device
        )
        if num_elements is not None:
            lengths = torch.as_tensor(
                num_elements, dtype=torch.long, device=self.device
            )
            if lengths.ndim == 0:
                lengths = lengths.repeat(batch_size)
            processed["mask"] = (
                torch.arange(self.processor.max_num_comp, device=self.device)[
                    None, :
                ]
                < lengths[:, None]
            )
    else:
        _require_condition_inputs(
            condition_type=condition_type, bbox=bbox, labels=labels
        )
        processed = self.processor(
            bbox=bbox,
            labels=labels,
            mask=mask,
            box_format=box_format,
            normalized=normalized,
            canvas_size=canvas_size,
            device=self.device,
        )
        batch_size = processed["box"].shape[0]
    mask_box, mask_cat = self.processor.condition_masks(
        str(canonical), mask=processed["mask"]
    )
    sample = {
        "box": processed["box"],
        "box_cond": processed["box_cond"],
        "cat": processed["cat"],
        "mask_box": mask_box,
        "mask_cat": mask_cat,
    }
    noisy_batch = {
        "box": torch.randn(
            processed["box"].shape,
            dtype=processed["box"].dtype,
            device=self.device,
            generator=generator,
        ),
        "cat": torch.full(
            processed["cat"].shape,
            self.processor.mask_category_id,
            dtype=torch.long,
            device=self.device,
        ),
    }
    original_temperature = self.scheduler.temperature
    if temperature is not None:
        self.scheduler.temperature = temperature
    steps = max(1, num_inference_steps or self.scheduler.num_train_timesteps)
    trajectory: list[Float[torch.Tensor, "batch elements 4"]] | None = (
        [] if return_intermediates else None
    )
    bbox_step: DLTJointSchedulerOutput | None = None
    try:
        for i in range(steps - 1, -1, -1):
            t_value = min(i, self.scheduler.num_train_timesteps - 1)
            t = torch.tensor([t_value] * batch_size, device=self.device)
            bbox_pred, cat_pred = self.model(sample, noisy_batch, timesteps=t)
            bbox_step, cat_step = self.scheduler.step_jointly(
                bbox_pred,
                {"cat": cat_pred},
                timestep=t,
                sample=noisy_batch["box"],
                generator=generator,
            )
            noisy_batch["box"] = bbox_step.prev_sample
            noisy_batch["cat"] = cat_step["cat"]
            if trajectory is not None:
                trajectory.append(noisy_batch["box"].detach().cpu())
    finally:
        self.scheduler.temperature = original_temperature
    if bbox_step is None:
        raise RuntimeError("DLT denoising did not run any scheduler steps")

    final_box = (
        sample["mask_box"] * bbox_step.pred_original_sample
        + (1 - sample["mask_box"]) * sample["box_cond"]
    )
    final_cat = (
        sample["mask_cat"] * noisy_batch["cat"]
        + (1 - sample["mask_cat"]) * sample["cat"]
    )
    valid_mask = processed["mask"].detach().cpu()
    output = LayoutGenerationOutput(
        bbox=self.processor.internal_to_public_boxes(final_box).detach().cpu()
        * valid_mask.unsqueeze(-1),
        labels=self.processor.internal_to_public_labels(
            final_cat, processed["mask"]
        )
        .detach()
        .cpu(),
        mask=valid_mask,
        id2label=self.processor.id2label,
        trajectory=trajectory,
        intermediates={"condition_type": str(canonical)}
        if return_intermediates
        else None,
    )
    return _format_pipeline_output(output, output_kind)

save_pretrained

save_pretrained(save_directory: str | Path) -> None

Persist DLT model, scheduler, and pipeline metadata.

Source code in models/dlt/src/dlt/pipeline_dlt.py
325
326
327
328
def save_pretrained(self, save_directory: str | Path) -> None:
    """Persist DLT model, scheduler, and pipeline metadata."""
    super().save_pretrained(save_directory, safe_serialization=False)
    self.dlt_config.save_config(save_directory)

from_pretrained classmethod

from_pretrained(
    pretrained_model_name_or_path: str | Path,
) -> Self

Load a saved DLT pipeline.

Source code in models/dlt/src/dlt/pipeline_dlt.py
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
@classmethod
def from_pretrained(cls, pretrained_model_name_or_path: str | Path) -> Self:
    """Load a saved DLT pipeline."""
    config_dict, _ = DLTConfig.load_config(
        pretrained_model_name_or_path, return_unused_kwargs=True
    )
    config = cast(DLTConfig, DLTConfig.from_config(config_dict))
    pipe = super().from_pretrained(pretrained_model_name_or_path, config=config)
    pipe.dlt_config = config
    pipe.processor = DLTProcessor(
        dataset=config.dataset_name,
        labels=tuple(config.id2label.values()),
        max_num_comp=config.max_num_comp,
    )
    return pipe

OutputType

Bases: StrEnum

DLT pipeline output containers.

Source code in models/dlt/src/dlt/pipeline_dlt.py
21
22
23
24
25
class OutputType(StrEnum):
    """DLT pipeline output containers."""

    dataclass = auto()
    dict = auto()

DLTProcessor

Bases: ProcessorMixin

Encode DLT public inputs into the package tensor format.

Parameters:

Name Type Description Default
dataset DatasetName | str

Canonical dataset name.

required
labels Sequence[str]

Ordered public labels without internal pad/drop ids.

required
max_num_comp int

Maximum number of layout elements.

required
Source code in models/dlt/src/dlt/processing_dlt.py
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
class DLTProcessor(ProcessorMixin):
    """Encode DLT public inputs into the package tensor format.

    Args:
        dataset: Canonical dataset name.
        labels: Ordered public labels without internal pad/drop ids.
        max_num_comp: Maximum number of layout elements.
    """

    config_name = "processor_config.json"

    def __init__(
        self,
        dataset: DatasetName | str,
        labels: Sequence[str],
        max_num_comp: int,
    ) -> None:
        """Initialize processor metadata."""
        super().__init__()
        self.dataset = str(normalize_dataset(dataset))
        self.labels = tuple(str(label) for label in labels)
        self.max_num_comp = max_num_comp

    @classmethod
    def from_dataset(cls, dataset: DatasetName | str) -> "DLTProcessor":
        """Create a processor from shared dataset metadata."""
        canonical = normalize_dataset(dataset)
        return cls(
            dataset=canonical,
            labels=tuple(default_id2label(canonical).values()),
            max_num_comp=9
            if canonical in {DatasetName.publaynet, DatasetName.rico13}
            else 33,
        )

    @property
    def id2label(self) -> dict[int, str]:
        """Return public label names keyed by dataset-local ids."""
        return dict(enumerate(self.labels))

    @property
    def categories_num(self) -> int:
        """Return internal category count including pad and mask/drop ids."""
        return len(self.labels) + 2

    @property
    def pad_category_id(self) -> int:
        """Return the internal padding category id."""
        return 0

    @property
    def mask_category_id(self) -> int:
        """Return the internal mask/drop category id."""
        return self.categories_num - 1

    def __call__(
        self,
        *,
        bbox: Float[torch.Tensor, "batch elements 4"]
        | Float[np.ndarray, "batch elements 4"]
        | LayoutInput,
        labels: Int[torch.Tensor, "batch elements"]
        | Int[np.ndarray, "batch elements"]
        | LayoutInput,
        mask: Bool[torch.Tensor, "batch elements"]
        | Bool[np.ndarray, "batch elements"]
        | LayoutInput
        | None = None,
        box_format: BoxFormat | str = BoxFormat.xywh,
        normalized: bool = True,
        canvas_size: tuple[int, int] | None = None,
        device: torch.device | str | None = None,
    ) -> DLTProcessedBatch:
        """Convert public layout tensors into padded internal tensors."""
        bbox_t, labels_t, mask_t = prepare_layout_tensors(
            bbox=bbox,
            labels=labels,
            mask=mask,
            box_format=box_format,
            normalized=normalized,
            canvas_size=canvas_size,
        )
        if device is not None:
            bbox_t = bbox_t.to(device)
            labels_t = labels_t.to(device)
            mask_t = mask_t.to(device)
        bbox_t, labels_t, mask_t = self.pad(bbox_t, labels_t, mask_t)
        return cast(
            DLTProcessedBatch,
            {
                "box": self.public_to_internal_boxes(bbox_t) * mask_t.unsqueeze(-1),
                "box_cond": self.public_to_internal_boxes(bbox_t)
                * mask_t.unsqueeze(-1),
                "cat": self.public_to_internal_labels(labels_t, mask_t),
                "mask": mask_t,
            },
        )

    def empty_condition(
        self,
        *,
        batch_size: int,
        device: torch.device | str,
        dtype: torch.dtype = torch.float32,
    ) -> DLTProcessedBatch:
        """Return an empty unconditional internal batch."""
        device = torch.device(device)
        bbox = torch.zeros(batch_size, self.max_num_comp, 4, dtype=dtype, device=device)
        labels = torch.zeros(
            batch_size, self.max_num_comp, dtype=torch.long, device=device
        )
        mask = torch.ones(
            batch_size, self.max_num_comp, dtype=torch.bool, device=device
        )
        return cast(
            DLTProcessedBatch,
            {
                "box": self.public_to_internal_boxes(bbox),
                "box_cond": self.public_to_internal_boxes(bbox),
                "cat": self.public_to_internal_labels(labels, mask),
                "mask": mask,
            },
        )

    def pad(
        self,
        bbox: Float[torch.Tensor, "batch elements 4"],
        labels: Int[torch.Tensor, "batch elements"],
        mask: Bool[torch.Tensor, "batch elements"] | None = None,
    ) -> tuple[
        Float[torch.Tensor, "batch max_elements 4"],
        Int[torch.Tensor, "batch max_elements"],
        Bool[torch.Tensor, "batch max_elements"],
    ]:
        """Pad a layout batch to ``max_num_comp``."""
        if bbox.shape[1] > self.max_num_comp:
            raise ValueError(f"DLT supports at most {self.max_num_comp} elements")

        if mask is None:
            mask = torch.ones(labels.shape, dtype=torch.bool, device=labels.device)
        pad_count = self.max_num_comp - bbox.shape[1]
        if pad_count:
            bbox = torch.cat(
                [
                    bbox,
                    torch.zeros(
                        bbox.shape[0],
                        pad_count,
                        4,
                        dtype=bbox.dtype,
                        device=bbox.device,
                    ),
                ],
                dim=1,
            )
            labels = torch.cat(
                [
                    labels,
                    torch.zeros(
                        labels.shape[0],
                        pad_count,
                        dtype=labels.dtype,
                        device=labels.device,
                    ),
                ],
                dim=1,
            )
            mask = torch.cat(
                [
                    mask,
                    torch.zeros(
                        mask.shape[0], pad_count, dtype=torch.bool, device=mask.device
                    ),
                ],
                dim=1,
            )
        return bbox, labels, mask

    def public_to_internal_boxes(
        self, bbox: Float[torch.Tensor, "batch elements 4"]
    ) -> Float[torch.Tensor, "batch elements 4"]:
        """Map public normalized ``xywh`` boxes to DLT's internal range."""
        return bbox.clamp(0.0, 1.0) * 4.0 - 2.0

    def internal_to_public_boxes(
        self, bbox: Float[torch.Tensor, "batch elements 4"]
    ) -> Float[torch.Tensor, "batch elements 4"]:
        """Map DLT internal-range boxes to public normalized ``xywh``."""
        return (bbox / 2.0 + 1.0).div(2.0).clamp(0.0, 1.0)

    def public_to_internal_labels(
        self,
        labels: Int[torch.Tensor, "batch elements"],
        mask: Bool[torch.Tensor, "batch elements"],
    ) -> Int[torch.Tensor, "batch elements"]:
        """Shift public labels into DLT's internal category ids."""
        shifted = labels.long() + 1
        return torch.where(mask.bool(), shifted, torch.zeros_like(shifted))

    def internal_to_public_labels(
        self,
        labels: Int[torch.Tensor, "batch elements"],
        mask: Bool[torch.Tensor, "batch elements"],
    ) -> Int[torch.Tensor, "batch elements"]:
        """Shift DLT internal category ids back to public dataset-local ids."""
        public = (labels.long() - 1).clamp(0, len(self.labels) - 1)
        return public * mask.long()

    def condition_masks(
        self, condition_type: str, *, mask: Bool[torch.Tensor, "batch elements"]
    ) -> tuple[
        Int[torch.Tensor, "batch elements 4"],
        Int[torch.Tensor, "batch elements"],
    ]:
        """Return DLT ``mask_box`` and ``mask_cat`` tensors.

        ``1`` means generated/noised and ``0`` means conditioned.
        """
        mask_box = torch.ones(
            mask.shape[0], mask.shape[1], 4, dtype=torch.long, device=mask.device
        )
        mask_cat = torch.ones(mask.shape, dtype=torch.long, device=mask.device)
        if condition_type == "label":
            mask_cat.zero_()
        elif condition_type == "label_size":
            mask_box[:, :, 2:] = 0
            mask_cat.zero_()
        elif condition_type == "unconditional":
            pass
        else:
            raise ValueError(f"Unsupported DLT condition_type: {condition_type}")

        mask_box = mask_box * mask.unsqueeze(-1).long()
        mask_cat = mask_cat * mask.long()
        return mask_box, mask_cat

id2label property

id2label: dict[int, str]

Return public label names keyed by dataset-local ids.

categories_num property

categories_num: int

Return internal category count including pad and mask/drop ids.

pad_category_id property

pad_category_id: int

Return the internal padding category id.

mask_category_id property

mask_category_id: int

Return the internal mask/drop category id.

__init__

__init__(
    dataset: DatasetName | str,
    labels: Sequence[str],
    max_num_comp: int,
) -> None

Initialize processor metadata.

Source code in models/dlt/src/dlt/processing_dlt.py
43
44
45
46
47
48
49
50
51
52
53
def __init__(
    self,
    dataset: DatasetName | str,
    labels: Sequence[str],
    max_num_comp: int,
) -> None:
    """Initialize processor metadata."""
    super().__init__()
    self.dataset = str(normalize_dataset(dataset))
    self.labels = tuple(str(label) for label in labels)
    self.max_num_comp = max_num_comp

from_dataset classmethod

from_dataset(dataset: DatasetName | str) -> DLTProcessor

Create a processor from shared dataset metadata.

Source code in models/dlt/src/dlt/processing_dlt.py
55
56
57
58
59
60
61
62
63
64
65
@classmethod
def from_dataset(cls, dataset: DatasetName | str) -> "DLTProcessor":
    """Create a processor from shared dataset metadata."""
    canonical = normalize_dataset(dataset)
    return cls(
        dataset=canonical,
        labels=tuple(default_id2label(canonical).values()),
        max_num_comp=9
        if canonical in {DatasetName.publaynet, DatasetName.rico13}
        else 33,
    )

__call__

__call__(
    *,
    bbox: Float[Tensor, "batch elements 4"]
    | Float[ndarray, "batch elements 4"]
    | LayoutInput,
    labels: Int[Tensor, "batch elements"]
    | Int[ndarray, "batch elements"]
    | LayoutInput,
    mask: Bool[Tensor, "batch elements"]
    | Bool[ndarray, "batch elements"]
    | LayoutInput
    | None = None,
    box_format: BoxFormat | str = BoxFormat.xywh,
    normalized: bool = True,
    canvas_size: tuple[int, int] | None = None,
    device: device | str | None = None,
) -> DLTProcessedBatch

Convert public layout tensors into padded internal tensors.

Source code in models/dlt/src/dlt/processing_dlt.py
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
def __call__(
    self,
    *,
    bbox: Float[torch.Tensor, "batch elements 4"]
    | Float[np.ndarray, "batch elements 4"]
    | LayoutInput,
    labels: Int[torch.Tensor, "batch elements"]
    | Int[np.ndarray, "batch elements"]
    | LayoutInput,
    mask: Bool[torch.Tensor, "batch elements"]
    | Bool[np.ndarray, "batch elements"]
    | LayoutInput
    | None = None,
    box_format: BoxFormat | str = BoxFormat.xywh,
    normalized: bool = True,
    canvas_size: tuple[int, int] | None = None,
    device: torch.device | str | None = None,
) -> DLTProcessedBatch:
    """Convert public layout tensors into padded internal tensors."""
    bbox_t, labels_t, mask_t = prepare_layout_tensors(
        bbox=bbox,
        labels=labels,
        mask=mask,
        box_format=box_format,
        normalized=normalized,
        canvas_size=canvas_size,
    )
    if device is not None:
        bbox_t = bbox_t.to(device)
        labels_t = labels_t.to(device)
        mask_t = mask_t.to(device)
    bbox_t, labels_t, mask_t = self.pad(bbox_t, labels_t, mask_t)
    return cast(
        DLTProcessedBatch,
        {
            "box": self.public_to_internal_boxes(bbox_t) * mask_t.unsqueeze(-1),
            "box_cond": self.public_to_internal_boxes(bbox_t)
            * mask_t.unsqueeze(-1),
            "cat": self.public_to_internal_labels(labels_t, mask_t),
            "mask": mask_t,
        },
    )

empty_condition

empty_condition(
    *,
    batch_size: int,
    device: device | str,
    dtype: dtype = torch.float32,
) -> DLTProcessedBatch

Return an empty unconditional internal batch.

Source code in models/dlt/src/dlt/processing_dlt.py
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
def empty_condition(
    self,
    *,
    batch_size: int,
    device: torch.device | str,
    dtype: torch.dtype = torch.float32,
) -> DLTProcessedBatch:
    """Return an empty unconditional internal batch."""
    device = torch.device(device)
    bbox = torch.zeros(batch_size, self.max_num_comp, 4, dtype=dtype, device=device)
    labels = torch.zeros(
        batch_size, self.max_num_comp, dtype=torch.long, device=device
    )
    mask = torch.ones(
        batch_size, self.max_num_comp, dtype=torch.bool, device=device
    )
    return cast(
        DLTProcessedBatch,
        {
            "box": self.public_to_internal_boxes(bbox),
            "box_cond": self.public_to_internal_boxes(bbox),
            "cat": self.public_to_internal_labels(labels, mask),
            "mask": mask,
        },
    )

pad

pad(
    bbox: Float[Tensor, "batch elements 4"],
    labels: Int[Tensor, "batch elements"],
    mask: Bool[Tensor, "batch elements"] | None = None,
) -> tuple[
    Float[torch.Tensor, "batch max_elements 4"],
    Int[torch.Tensor, "batch max_elements"],
    Bool[torch.Tensor, "batch max_elements"],
]

Pad a layout batch to max_num_comp.

Source code in models/dlt/src/dlt/processing_dlt.py
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
def pad(
    self,
    bbox: Float[torch.Tensor, "batch elements 4"],
    labels: Int[torch.Tensor, "batch elements"],
    mask: Bool[torch.Tensor, "batch elements"] | None = None,
) -> tuple[
    Float[torch.Tensor, "batch max_elements 4"],
    Int[torch.Tensor, "batch max_elements"],
    Bool[torch.Tensor, "batch max_elements"],
]:
    """Pad a layout batch to ``max_num_comp``."""
    if bbox.shape[1] > self.max_num_comp:
        raise ValueError(f"DLT supports at most {self.max_num_comp} elements")

    if mask is None:
        mask = torch.ones(labels.shape, dtype=torch.bool, device=labels.device)
    pad_count = self.max_num_comp - bbox.shape[1]
    if pad_count:
        bbox = torch.cat(
            [
                bbox,
                torch.zeros(
                    bbox.shape[0],
                    pad_count,
                    4,
                    dtype=bbox.dtype,
                    device=bbox.device,
                ),
            ],
            dim=1,
        )
        labels = torch.cat(
            [
                labels,
                torch.zeros(
                    labels.shape[0],
                    pad_count,
                    dtype=labels.dtype,
                    device=labels.device,
                ),
            ],
            dim=1,
        )
        mask = torch.cat(
            [
                mask,
                torch.zeros(
                    mask.shape[0], pad_count, dtype=torch.bool, device=mask.device
                ),
            ],
            dim=1,
        )
    return bbox, labels, mask

public_to_internal_boxes

public_to_internal_boxes(
    bbox: Float[Tensor, "batch elements 4"],
) -> Float[torch.Tensor, "batch elements 4"]

Map public normalized xywh boxes to DLT's internal range.

Source code in models/dlt/src/dlt/processing_dlt.py
210
211
212
213
214
def public_to_internal_boxes(
    self, bbox: Float[torch.Tensor, "batch elements 4"]
) -> Float[torch.Tensor, "batch elements 4"]:
    """Map public normalized ``xywh`` boxes to DLT's internal range."""
    return bbox.clamp(0.0, 1.0) * 4.0 - 2.0

internal_to_public_boxes

internal_to_public_boxes(
    bbox: Float[Tensor, "batch elements 4"],
) -> Float[torch.Tensor, "batch elements 4"]

Map DLT internal-range boxes to public normalized xywh.

Source code in models/dlt/src/dlt/processing_dlt.py
216
217
218
219
220
def internal_to_public_boxes(
    self, bbox: Float[torch.Tensor, "batch elements 4"]
) -> Float[torch.Tensor, "batch elements 4"]:
    """Map DLT internal-range boxes to public normalized ``xywh``."""
    return (bbox / 2.0 + 1.0).div(2.0).clamp(0.0, 1.0)

public_to_internal_labels

public_to_internal_labels(
    labels: Int[Tensor, "batch elements"],
    mask: Bool[Tensor, "batch elements"],
) -> Int[torch.Tensor, "batch elements"]

Shift public labels into DLT's internal category ids.

Source code in models/dlt/src/dlt/processing_dlt.py
222
223
224
225
226
227
228
229
def public_to_internal_labels(
    self,
    labels: Int[torch.Tensor, "batch elements"],
    mask: Bool[torch.Tensor, "batch elements"],
) -> Int[torch.Tensor, "batch elements"]:
    """Shift public labels into DLT's internal category ids."""
    shifted = labels.long() + 1
    return torch.where(mask.bool(), shifted, torch.zeros_like(shifted))

internal_to_public_labels

internal_to_public_labels(
    labels: Int[Tensor, "batch elements"],
    mask: Bool[Tensor, "batch elements"],
) -> Int[torch.Tensor, "batch elements"]

Shift DLT internal category ids back to public dataset-local ids.

Source code in models/dlt/src/dlt/processing_dlt.py
231
232
233
234
235
236
237
238
def internal_to_public_labels(
    self,
    labels: Int[torch.Tensor, "batch elements"],
    mask: Bool[torch.Tensor, "batch elements"],
) -> Int[torch.Tensor, "batch elements"]:
    """Shift DLT internal category ids back to public dataset-local ids."""
    public = (labels.long() - 1).clamp(0, len(self.labels) - 1)
    return public * mask.long()

condition_masks

condition_masks(
    condition_type: str,
    *,
    mask: Bool[Tensor, "batch elements"],
) -> tuple[
    Int[torch.Tensor, "batch elements 4"],
    Int[torch.Tensor, "batch elements"],
]

Return DLT mask_box and mask_cat tensors.

1 means generated/noised and 0 means conditioned.

Source code in models/dlt/src/dlt/processing_dlt.py
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
def condition_masks(
    self, condition_type: str, *, mask: Bool[torch.Tensor, "batch elements"]
) -> tuple[
    Int[torch.Tensor, "batch elements 4"],
    Int[torch.Tensor, "batch elements"],
]:
    """Return DLT ``mask_box`` and ``mask_cat`` tensors.

    ``1`` means generated/noised and ``0`` means conditioned.
    """
    mask_box = torch.ones(
        mask.shape[0], mask.shape[1], 4, dtype=torch.long, device=mask.device
    )
    mask_cat = torch.ones(mask.shape, dtype=torch.long, device=mask.device)
    if condition_type == "label":
        mask_cat.zero_()
    elif condition_type == "label_size":
        mask_box[:, :, 2:] = 0
        mask_cat.zero_()
    elif condition_type == "unconditional":
        pass
    else:
        raise ValueError(f"Unsupported DLT condition_type: {condition_type}")

    mask_box = mask_box * mask.unsqueeze(-1).long()
    mask_cat = mask_cat * mask.long()
    return mask_box, mask_cat

DLTJointDiffusionScheduler

Bases: SchedulerMixin, ConfigMixin

Save/loadable DLT continuous and discrete diffusion scheduler.

Parameters:

Name Type Description Default
alpha float

Probability of changing to a non-mask category.

0.0
beta float

Probability of changing to the mask/drop category.

0.15
seq_max_length int

Maximum number of layout elements.

9
discrete_features_names Sequence[Sequence[str | int]] | None

Discrete feature specs as (name, count).

None
num_discrete_steps Sequence[int] | None

Number of discrete diffusion steps per feature.

None
temperature float

Categorical sampling temperature.

0.8
num_train_timesteps int

Continuous DDPM timesteps.

100
beta_schedule str

Diffusers DDPM beta schedule.

'squaredcos_cap_v2'
prediction_type str

DDPM prediction type.

'sample'
clip_sample bool

Whether DDPM steps clamp predicted samples.

False
Source code in models/dlt/src/dlt/scheduling_dlt.py
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
class DLTJointDiffusionScheduler(SchedulerMixin, ConfigMixin):
    """Save/loadable DLT continuous and discrete diffusion scheduler.

    Args:
        alpha: Probability of changing to a non-mask category.
        beta: Probability of changing to the mask/drop category.
        seq_max_length: Maximum number of layout elements.
        discrete_features_names: Discrete feature specs as ``(name, count)``.
        num_discrete_steps: Number of discrete diffusion steps per feature.
        temperature: Categorical sampling temperature.
        num_train_timesteps: Continuous DDPM timesteps.
        beta_schedule: Diffusers DDPM beta schedule.
        prediction_type: DDPM prediction type.
        clip_sample: Whether DDPM steps clamp predicted samples.
    """

    config_name = "scheduler_config.json"
    order = 1

    @register_to_config
    def __init__(
        self,
        *,
        alpha: float = 0.0,
        beta: float = 0.15,
        seq_max_length: int = 9,
        discrete_features_names: Sequence[Sequence[str | int]] | None = None,
        num_discrete_steps: Sequence[int] | None = None,
        temperature: float = 0.8,
        num_train_timesteps: int = 100,
        beta_schedule: str = "squaredcos_cap_v2",
        prediction_type: str = "sample",
        clip_sample: bool = False,
    ) -> None:
        """Initialize the scheduler and defer transition matrix construction."""
        features = discrete_features_names or _DEFAULT_FEATURES
        steps = list(num_discrete_steps or [10 for _ in features])
        if len(features) != len(steps):
            raise ValueError("Each discrete feature requires a step count")

        self.alpha = alpha
        self.beta = beta
        self.seq_max_length = seq_max_length
        parsed_features: list[DiscreteFeatureSpec] = []
        for raw_feature in features:
            name = str(raw_feature[0])
            raw_count = raw_feature[1]
            if isinstance(raw_count, int):
                count = raw_count
            else:
                count = int(str(raw_count))
            parsed_features.append((name, count))
        self.discrete_features_names = parsed_features
        self.num_discrete_steps = [int(step) for step in steps]
        self.temperature = temperature
        self._cont2disc: dict[str, dict[int, int]] | None = None
        self._transition_matrices: (
            dict[str, list[Float[torch.Tensor, "categories categories"]]] | None
        ) = None
        self._ddpm = DDPMScheduler(
            num_train_timesteps=num_train_timesteps,
            beta_schedule=beta_schedule,
            prediction_type=prediction_type,
            clip_sample=clip_sample,
        )
        self.num_cont_steps = num_train_timesteps
        self.num_train_timesteps = num_train_timesteps
        self.beta_schedule = beta_schedule
        self.prediction_type = prediction_type
        self.clip_sample = clip_sample

    @property
    def cont2disc(self) -> dict[str, dict[int, int]]:
        """Return continuous-to-discrete timestep mappings, computing lazily."""
        if self._cont2disc is None:
            self._cont2disc = {
                name: self.mapping_cont2disc(self.num_train_timesteps, steps)
                for (name, _), steps in zip(
                    self.discrete_features_names, self.num_discrete_steps, strict=True
                )
            }
        return self._cont2disc

    @property
    def transition_matrices(
        self,
    ) -> dict[str, list[Float[torch.Tensor, "categories categories"]]]:
        """Return discrete transition matrices, computing lazily."""
        if self._transition_matrices is None:
            self._transition_matrices = {
                name: self.generate_transition_mat(count, steps)
                for (name, count), steps in zip(
                    self.discrete_features_names, self.num_discrete_steps, strict=True
                )
            }
        return self._transition_matrices

    def add_noise_jointly(
        self,
        vec_cont: Float[torch.Tensor, "batch elements 4"],
        vec_cat: Mapping[str, Float[torch.Tensor, "..."] | Int[torch.Tensor, "..."]],
        timesteps: Int[torch.Tensor, "batch"],
        noise: Float[torch.Tensor, "batch elements 4"],
        generator: torch.Generator | None = None,
    ) -> tuple[
        Float[torch.Tensor, "batch elements 4"],
        dict[str, Int[torch.Tensor, "batch elements"]],
    ]:
        """Add continuous DDPM noise and discrete categorical noise."""
        noised_cont = self._ddpm.add_noise(
            original_samples=vec_cont,
            timesteps=cast(torch.IntTensor, timesteps),
            noise=noise,
        )
        cat_res: dict[str, Int[torch.Tensor, "batch elements"]] = {}
        for f_name, _ in self.discrete_features_names:
            t_to_discrete_stage = [
                self.cont2disc[f_name][int(t.item())] for t in timesteps
            ]
            prob_mat = [
                self.transition_matrices[f_name][u].to(vec_cont.device)[
                    vec_cat[f_name][i]
                ]
                for i, u in enumerate(t_to_discrete_stage)
            ]
            probs = torch.cat(prob_mat)
            cat_noise = torch.multinomial(
                probs, 1, replacement=True, generator=generator
            )
            cat_res[f_name] = rearrange(
                cat_noise, "(d b) 1 -> d b", d=noised_cont.shape[0]
            )
        return noised_cont, cat_res

    def step_jointly(
        self,
        cont_output: Float[torch.Tensor, "batch elements 4"],
        cat_output: dict[str, Float[torch.Tensor, "batch elements categories"]],
        timestep: Int[torch.Tensor, "batch"],
        sample: Float[torch.Tensor, "batch elements 4"],
        generator: torch.Generator | None = None,
        return_dict: bool = True,
    ) -> tuple[DLTJointSchedulerOutput, dict[str, Int[torch.Tensor, "batch elements"]]]:
        """Take one reverse step for boxes and categories."""
        bbox = cast(
            DDPMSchedulerOutput,
            self._ddpm.step(
                cont_output,
                int(timestep.flatten()[0].item()),
                sample,
                generator=generator,
                return_dict=True,
            ),
        )
        bbox_out = DLTJointSchedulerOutput(
            prev_sample=bbox.prev_sample,
            pred_original_sample=cast(torch.Tensor, bbox.pred_original_sample),
        )
        step_cat_res: dict[str, Int[torch.Tensor, "batch elements"]] = {}
        batch_timestep = (
            timestep
            if timestep.numel() == sample.shape[0]
            else timestep.flatten()[0].repeat(sample.shape[0])
        )
        for f_name, f_cat_num in self.discrete_features_names:
            t_to_discrete_stage = [
                self.cont2disc[f_name][int(t.item())] for t in batch_timestep
            ]
            cls, _ = self.denoise_cat(
                cat_output[f_name],
                t_to_discrete_stage,
                f_cat_num,
                self.transition_matrices[f_name],
                generator=generator,
            )
            step_cat_res[f_name] = cls
        return bbox_out, step_cat_res

    def generate_transition_mat(
        self, categories_num: int, num_discrete_steps: int
    ) -> list[Float[torch.Tensor, "categories categories"]]:
        """Generate Markov transition matrices for one discrete feature."""
        transition_mat = (
            np.eye(categories_num) * (1 - self.alpha - self.beta)
            + self.alpha / categories_num
        )
        transition_mat[:, -1] += self.beta
        transition_mat[-1, :] = 0
        transition_mat[-1, -1] = 1
        transition_mat_list: list[Float[torch.Tensor, "categories categories"]] = []
        curr_mat = transition_mat.copy()
        for _ in range(num_discrete_steps):
            transition_mat_list.append(torch.tensor(curr_mat, dtype=torch.float32))
            curr_mat = curr_mat @ transition_mat
        return transition_mat_list

    def denoise_cat(
        self,
        pred: Float[torch.Tensor, "batch elements categories"],
        t: list[int],
        cat_num: int,
        transition_mat_list: list[Float[torch.Tensor, "categories categories"]],
        generator: torch.Generator | None = None,
    ) -> tuple[Int[torch.Tensor, "batch elements"], int]:
        """Denoise a categorical feature using DLT's transition rule."""
        pred_prob = F.softmax(pred, dim=2)
        prob, cls = torch.max(pred_prob, dim=2)
        if t[0] > 1:
            matrix = transition_mat_list[t[0]].to(device=pred.device, dtype=pred.dtype)
            scores = torch.matmul(pred_prob.reshape((-1, cat_num)), matrix)
            scores = scores.reshape(pred_prob.shape)
            scores[:, :, 0] = 0
            logits = scores / self.temperature
            flat = logits.reshape(-1, cat_num)
            res = torch.multinomial(flat, 1, generator=generator).reshape(cls.shape)
        else:
            res = (cat_num - 1) * torch.ones_like(cls, dtype=torch.long)
            top = torch.topk(prob, prob.shape[1], dim=1)
            for row in range(prob.shape[0]):
                res[row, top.indices[row]] = cls[row, top.indices[row]]
        return res, 0

    @staticmethod
    def mapping_cont2disc(
        num_cont_steps: int, num_discrete_steps: int
    ) -> dict[int, int]:
        """Map continuous timesteps onto discrete diffusion stages."""
        block_size = num_cont_steps // num_discrete_steps
        cont2disc: dict[int, int] = {}
        for i in range(num_cont_steps):
            if i >= (num_discrete_steps - 1) * block_size:
                if (
                    num_cont_steps % num_discrete_steps != 0
                    and i >= num_discrete_steps * block_size
                ):
                    cont2disc[i] = num_discrete_steps - 1
                else:
                    cont2disc[i] = i // block_size
            else:
                cont2disc[i] = i // block_size
        return cont2disc

cont2disc property

cont2disc: dict[str, dict[int, int]]

Return continuous-to-discrete timestep mappings, computing lazily.

transition_matrices property

transition_matrices: dict[
    str, list[Float[Tensor, "categories categories"]]
]

Return discrete transition matrices, computing lazily.

__init__

__init__(
    *,
    alpha: float = 0.0,
    beta: float = 0.15,
    seq_max_length: int = 9,
    discrete_features_names: Sequence[Sequence[str | int]]
    | None = None,
    num_discrete_steps: Sequence[int] | None = None,
    temperature: float = 0.8,
    num_train_timesteps: int = 100,
    beta_schedule: str = "squaredcos_cap_v2",
    prediction_type: str = "sample",
    clip_sample: bool = False,
) -> None

Initialize the scheduler and defer transition matrix construction.

Source code in models/dlt/src/dlt/scheduling_dlt.py
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
@register_to_config
def __init__(
    self,
    *,
    alpha: float = 0.0,
    beta: float = 0.15,
    seq_max_length: int = 9,
    discrete_features_names: Sequence[Sequence[str | int]] | None = None,
    num_discrete_steps: Sequence[int] | None = None,
    temperature: float = 0.8,
    num_train_timesteps: int = 100,
    beta_schedule: str = "squaredcos_cap_v2",
    prediction_type: str = "sample",
    clip_sample: bool = False,
) -> None:
    """Initialize the scheduler and defer transition matrix construction."""
    features = discrete_features_names or _DEFAULT_FEATURES
    steps = list(num_discrete_steps or [10 for _ in features])
    if len(features) != len(steps):
        raise ValueError("Each discrete feature requires a step count")

    self.alpha = alpha
    self.beta = beta
    self.seq_max_length = seq_max_length
    parsed_features: list[DiscreteFeatureSpec] = []
    for raw_feature in features:
        name = str(raw_feature[0])
        raw_count = raw_feature[1]
        if isinstance(raw_count, int):
            count = raw_count
        else:
            count = int(str(raw_count))
        parsed_features.append((name, count))
    self.discrete_features_names = parsed_features
    self.num_discrete_steps = [int(step) for step in steps]
    self.temperature = temperature
    self._cont2disc: dict[str, dict[int, int]] | None = None
    self._transition_matrices: (
        dict[str, list[Float[torch.Tensor, "categories categories"]]] | None
    ) = None
    self._ddpm = DDPMScheduler(
        num_train_timesteps=num_train_timesteps,
        beta_schedule=beta_schedule,
        prediction_type=prediction_type,
        clip_sample=clip_sample,
    )
    self.num_cont_steps = num_train_timesteps
    self.num_train_timesteps = num_train_timesteps
    self.beta_schedule = beta_schedule
    self.prediction_type = prediction_type
    self.clip_sample = clip_sample

add_noise_jointly

add_noise_jointly(
    vec_cont: Float[Tensor, "batch elements 4"],
    vec_cat: Mapping[
        str, Float[Tensor, ...] | Int[Tensor, ...]
    ],
    timesteps: Int[Tensor, batch],
    noise: Float[Tensor, "batch elements 4"],
    generator: Generator | None = None,
) -> tuple[
    Float[torch.Tensor, "batch elements 4"],
    dict[str, Int[torch.Tensor, "batch elements"]],
]

Add continuous DDPM noise and discrete categorical noise.

Source code in models/dlt/src/dlt/scheduling_dlt.py
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
def add_noise_jointly(
    self,
    vec_cont: Float[torch.Tensor, "batch elements 4"],
    vec_cat: Mapping[str, Float[torch.Tensor, "..."] | Int[torch.Tensor, "..."]],
    timesteps: Int[torch.Tensor, "batch"],
    noise: Float[torch.Tensor, "batch elements 4"],
    generator: torch.Generator | None = None,
) -> tuple[
    Float[torch.Tensor, "batch elements 4"],
    dict[str, Int[torch.Tensor, "batch elements"]],
]:
    """Add continuous DDPM noise and discrete categorical noise."""
    noised_cont = self._ddpm.add_noise(
        original_samples=vec_cont,
        timesteps=cast(torch.IntTensor, timesteps),
        noise=noise,
    )
    cat_res: dict[str, Int[torch.Tensor, "batch elements"]] = {}
    for f_name, _ in self.discrete_features_names:
        t_to_discrete_stage = [
            self.cont2disc[f_name][int(t.item())] for t in timesteps
        ]
        prob_mat = [
            self.transition_matrices[f_name][u].to(vec_cont.device)[
                vec_cat[f_name][i]
            ]
            for i, u in enumerate(t_to_discrete_stage)
        ]
        probs = torch.cat(prob_mat)
        cat_noise = torch.multinomial(
            probs, 1, replacement=True, generator=generator
        )
        cat_res[f_name] = rearrange(
            cat_noise, "(d b) 1 -> d b", d=noised_cont.shape[0]
        )
    return noised_cont, cat_res

step_jointly

step_jointly(
    cont_output: Float[Tensor, "batch elements 4"],
    cat_output: dict[
        str, Float[Tensor, "batch elements categories"]
    ],
    timestep: Int[Tensor, batch],
    sample: Float[Tensor, "batch elements 4"],
    generator: Generator | None = None,
    return_dict: bool = True,
) -> tuple[
    DLTJointSchedulerOutput,
    dict[str, Int[torch.Tensor, "batch elements"]],
]

Take one reverse step for boxes and categories.

Source code in models/dlt/src/dlt/scheduling_dlt.py
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
def step_jointly(
    self,
    cont_output: Float[torch.Tensor, "batch elements 4"],
    cat_output: dict[str, Float[torch.Tensor, "batch elements categories"]],
    timestep: Int[torch.Tensor, "batch"],
    sample: Float[torch.Tensor, "batch elements 4"],
    generator: torch.Generator | None = None,
    return_dict: bool = True,
) -> tuple[DLTJointSchedulerOutput, dict[str, Int[torch.Tensor, "batch elements"]]]:
    """Take one reverse step for boxes and categories."""
    bbox = cast(
        DDPMSchedulerOutput,
        self._ddpm.step(
            cont_output,
            int(timestep.flatten()[0].item()),
            sample,
            generator=generator,
            return_dict=True,
        ),
    )
    bbox_out = DLTJointSchedulerOutput(
        prev_sample=bbox.prev_sample,
        pred_original_sample=cast(torch.Tensor, bbox.pred_original_sample),
    )
    step_cat_res: dict[str, Int[torch.Tensor, "batch elements"]] = {}
    batch_timestep = (
        timestep
        if timestep.numel() == sample.shape[0]
        else timestep.flatten()[0].repeat(sample.shape[0])
    )
    for f_name, f_cat_num in self.discrete_features_names:
        t_to_discrete_stage = [
            self.cont2disc[f_name][int(t.item())] for t in batch_timestep
        ]
        cls, _ = self.denoise_cat(
            cat_output[f_name],
            t_to_discrete_stage,
            f_cat_num,
            self.transition_matrices[f_name],
            generator=generator,
        )
        step_cat_res[f_name] = cls
    return bbox_out, step_cat_res

generate_transition_mat

generate_transition_mat(
    categories_num: int, num_discrete_steps: int
) -> list[Float[torch.Tensor, "categories categories"]]

Generate Markov transition matrices for one discrete feature.

Source code in models/dlt/src/dlt/scheduling_dlt.py
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
def generate_transition_mat(
    self, categories_num: int, num_discrete_steps: int
) -> list[Float[torch.Tensor, "categories categories"]]:
    """Generate Markov transition matrices for one discrete feature."""
    transition_mat = (
        np.eye(categories_num) * (1 - self.alpha - self.beta)
        + self.alpha / categories_num
    )
    transition_mat[:, -1] += self.beta
    transition_mat[-1, :] = 0
    transition_mat[-1, -1] = 1
    transition_mat_list: list[Float[torch.Tensor, "categories categories"]] = []
    curr_mat = transition_mat.copy()
    for _ in range(num_discrete_steps):
        transition_mat_list.append(torch.tensor(curr_mat, dtype=torch.float32))
        curr_mat = curr_mat @ transition_mat
    return transition_mat_list

denoise_cat

denoise_cat(
    pred: Float[Tensor, "batch elements categories"],
    t: list[int],
    cat_num: int,
    transition_mat_list: list[
        Float[Tensor, "categories categories"]
    ],
    generator: Generator | None = None,
) -> tuple[Int[torch.Tensor, "batch elements"], int]

Denoise a categorical feature using DLT's transition rule.

Source code in models/dlt/src/dlt/scheduling_dlt.py
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
def denoise_cat(
    self,
    pred: Float[torch.Tensor, "batch elements categories"],
    t: list[int],
    cat_num: int,
    transition_mat_list: list[Float[torch.Tensor, "categories categories"]],
    generator: torch.Generator | None = None,
) -> tuple[Int[torch.Tensor, "batch elements"], int]:
    """Denoise a categorical feature using DLT's transition rule."""
    pred_prob = F.softmax(pred, dim=2)
    prob, cls = torch.max(pred_prob, dim=2)
    if t[0] > 1:
        matrix = transition_mat_list[t[0]].to(device=pred.device, dtype=pred.dtype)
        scores = torch.matmul(pred_prob.reshape((-1, cat_num)), matrix)
        scores = scores.reshape(pred_prob.shape)
        scores[:, :, 0] = 0
        logits = scores / self.temperature
        flat = logits.reshape(-1, cat_num)
        res = torch.multinomial(flat, 1, generator=generator).reshape(cls.shape)
    else:
        res = (cat_num - 1) * torch.ones_like(cls, dtype=torch.long)
        top = torch.topk(prob, prob.shape[1], dim=1)
        for row in range(prob.shape[0]):
            res[row, top.indices[row]] = cls[row, top.indices[row]]
    return res, 0

mapping_cont2disc staticmethod

mapping_cont2disc(
    num_cont_steps: int, num_discrete_steps: int
) -> dict[int, int]

Map continuous timesteps onto discrete diffusion stages.

Source code in models/dlt/src/dlt/scheduling_dlt.py
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
@staticmethod
def mapping_cont2disc(
    num_cont_steps: int, num_discrete_steps: int
) -> dict[int, int]:
    """Map continuous timesteps onto discrete diffusion stages."""
    block_size = num_cont_steps // num_discrete_steps
    cont2disc: dict[int, int] = {}
    for i in range(num_cont_steps):
        if i >= (num_discrete_steps - 1) * block_size:
            if (
                num_cont_steps % num_discrete_steps != 0
                and i >= num_discrete_steps * block_size
            ):
                cont2disc[i] = num_discrete_steps - 1
            else:
                cont2disc[i] = i // block_size
        else:
            cont2disc[i] = i // block_size
    return cont2disc

build_pipeline

build_pipeline(config: DLTConfig) -> DLTPipeline

Build a randomly initialized DLT pipeline from a config.

Parameters:

Name Type Description Default
config DLTConfig

DLT configuration.

required

Returns:

Type Description
DLTPipeline

Pipeline with model, scheduler, and processor components.

Source code in models/dlt/src/dlt/conversion.py
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
def build_pipeline(config: DLTConfig) -> DLTPipeline:
    """Build a randomly initialized DLT pipeline from a config.

    Args:
        config: DLT configuration.

    Returns:
        Pipeline with model, scheduler, and processor components.
    """
    model = DLT(
        categories_num=config.categories_num,
        latent_dim=config.latent_dim,
        num_layers=config.num_layers,
        num_heads=config.num_heads,
        dropout_r=config.dropout_r,
        activation=config.activation,
        cond_emb_size=config.cond_emb_size,
        cat_emb_size=config.cat_emb_size,
    )
    scheduler = DLTJointDiffusionScheduler(
        alpha=0.0,
        seq_max_length=config.max_num_comp,
        discrete_features_names=[("cat", config.categories_num)],
        num_discrete_steps=[config.num_discrete_steps],
        num_train_timesteps=config.num_cont_timesteps,
        beta_schedule=config.beta_schedule,
        prediction_type="sample",
        clip_sample=False,
    )
    processor = DLTProcessor(
        dataset=config.dataset_name,
        labels=tuple(config.id2label.values()),
        max_num_comp=config.max_num_comp,
    )
    return DLTPipeline(
        model=model, scheduler=scheduler, config=config, processor=processor
    )

convert_save_pretrained_directory

convert_save_pretrained_directory(
    checkpoint_dir: str | Path,
    output_dir: str | Path,
    *,
    config: DLTConfig,
) -> DLTPipeline

Convert an original DLT save_pretrained directory into a pipeline.

Parameters:

Name Type Description Default
checkpoint_dir str | Path

Directory containing the original DLT model files.

required
output_dir str | Path

Destination directory for the converted pipeline.

required
config DLTConfig

Dataset and scheduler metadata for the checkpoint.

required

Returns:

Type Description
DLTPipeline

The saved converted pipeline.

Raises:

Type Description
RuntimeError

If the checkpoint does not match the configured model.

Source code in models/dlt/src/dlt/conversion.py
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
def convert_save_pretrained_directory(
    checkpoint_dir: str | Path,
    output_dir: str | Path,
    *,
    config: DLTConfig,
) -> DLTPipeline:
    """Convert an original DLT ``save_pretrained`` directory into a pipeline.

    Args:
        checkpoint_dir: Directory containing the original DLT model files.
        output_dir: Destination directory for the converted pipeline.
        config: Dataset and scheduler metadata for the checkpoint.

    Returns:
        The saved converted pipeline.

    Raises:
        RuntimeError: If the checkpoint does not match the configured model.
    """
    pipe = build_pipeline(config)
    loaded_model = DLT.from_pretrained(checkpoint_dir)
    missing, unexpected = pipe.model.load_state_dict(
        loaded_model.state_dict(), strict=True
    )
    if missing or unexpected:
        raise RuntimeError(f"Missing keys: {missing}; unexpected keys: {unexpected}")

    pipe.save_pretrained(output_dir)
    return pipe

configuration_dlt

Configuration and dataset metadata for DLT pipelines.

DLTCoordinateRange

Bases: StrEnum

Closed set of DLT public coordinate ranges.

Source code in models/dlt/src/dlt/configuration_dlt.py
17
18
19
20
class DLTCoordinateRange(StrEnum):
    """Closed set of DLT public coordinate ranges."""

    normalized_0_1 = auto()

DLTConfig

Bases: ConfigMixin

Pipeline-level DLT configuration persisted with converted checkpoints.

Parameters:

Name Type Description Default
dataset_name str

Canonical dataset name.

'publaynet'
id2label dict[int | str, str] | None

Optional public label mapping. When omitted, shared dataset labels are used.

None
max_num_comp int | None

Maximum number of layout elements.

None
categories_num int | None

Internal category count including pad and mask/drop ids.

None
latent_dim int

Transformer latent dimension.

512
num_layers int

Number of transformer encoder layers.

4
num_heads int

Number of attention heads.

8
dropout_r float

Dropout probability.

0.0
activation str

Transformer activation.

'gelu'
cond_emb_size int

Box-condition embedding size.

224
cat_emb_size int

Category embedding size.

64
num_cont_timesteps int

Continuous DDPM training timesteps.

100
num_discrete_steps int

Discrete category diffusion steps.

10
beta_schedule str

DDPM beta schedule.

'squaredcos_cap_v2'
coordinate_range DLTCoordinateRange | str

Public coordinate range.

normalized_0_1
Source code in models/dlt/src/dlt/configuration_dlt.py
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
class DLTConfig(ConfigMixin):
    """Pipeline-level DLT configuration persisted with converted checkpoints.

    Args:
        dataset_name: Canonical dataset name.
        id2label: Optional public label mapping. When omitted, shared dataset
            labels are used.
        max_num_comp: Maximum number of layout elements.
        categories_num: Internal category count including pad and mask/drop ids.
        latent_dim: Transformer latent dimension.
        num_layers: Number of transformer encoder layers.
        num_heads: Number of attention heads.
        dropout_r: Dropout probability.
        activation: Transformer activation.
        cond_emb_size: Box-condition embedding size.
        cat_emb_size: Category embedding size.
        num_cont_timesteps: Continuous DDPM training timesteps.
        num_discrete_steps: Discrete category diffusion steps.
        beta_schedule: DDPM beta schedule.
        coordinate_range: Public coordinate range.
    """

    config_name = "dlt_config.json"

    @register_to_config
    def __init__(
        self,
        *,
        dataset_name: str = "publaynet",
        id2label: dict[int | str, str] | None = None,
        max_num_comp: int | None = None,
        categories_num: int | None = None,
        latent_dim: int = 512,
        num_layers: int = 4,
        num_heads: int = 8,
        dropout_r: float = 0.0,
        activation: str = "gelu",
        cond_emb_size: int = 224,
        cat_emb_size: int = 64,
        num_cont_timesteps: int = 100,
        num_discrete_steps: int = 10,
        beta_schedule: str = "squaredcos_cap_v2",
        coordinate_range: DLTCoordinateRange | str = DLTCoordinateRange.normalized_0_1,
    ) -> None:
        """Initialize DLT configuration."""
        dataset = normalize_dataset(dataset_name)
        labels = default_id2label(dataset) if id2label is None else id2label

        self.dataset_name = str(dataset)
        self.id2label = {int(key): value for key, value in labels.items()}
        self.max_num_comp = max_num_comp or max_elements_for_dataset(dataset)
        self.categories_num = categories_num or len(self.id2label) + 2

        self.latent_dim = latent_dim
        self.num_layers = num_layers
        self.num_heads = num_heads
        self.dropout_r = dropout_r
        self.activation = activation
        self.cond_emb_size = cond_emb_size
        self.cat_emb_size = cat_emb_size
        self.num_cont_timesteps = num_cont_timesteps
        self.num_discrete_steps = num_discrete_steps
        self.beta_schedule = beta_schedule
        self.coordinate_range = str(DLTCoordinateRange(coordinate_range))

    @property
    def label2id(self) -> dict[str, int]:
        """Return the public label-name to label-id mapping."""
        return {label: idx for idx, label in self.id2label.items()}

label2id property

label2id: dict[str, int]

Return the public label-name to label-id mapping.

__init__

__init__(
    *,
    dataset_name: str = "publaynet",
    id2label: dict[int | str, str] | None = None,
    max_num_comp: int | None = None,
    categories_num: int | None = None,
    latent_dim: int = 512,
    num_layers: int = 4,
    num_heads: int = 8,
    dropout_r: float = 0.0,
    activation: str = "gelu",
    cond_emb_size: int = 224,
    cat_emb_size: int = 64,
    num_cont_timesteps: int = 100,
    num_discrete_steps: int = 10,
    beta_schedule: str = "squaredcos_cap_v2",
    coordinate_range: DLTCoordinateRange
    | str = DLTCoordinateRange.normalized_0_1,
) -> None

Initialize DLT configuration.

Source code in models/dlt/src/dlt/configuration_dlt.py
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
@register_to_config
def __init__(
    self,
    *,
    dataset_name: str = "publaynet",
    id2label: dict[int | str, str] | None = None,
    max_num_comp: int | None = None,
    categories_num: int | None = None,
    latent_dim: int = 512,
    num_layers: int = 4,
    num_heads: int = 8,
    dropout_r: float = 0.0,
    activation: str = "gelu",
    cond_emb_size: int = 224,
    cat_emb_size: int = 64,
    num_cont_timesteps: int = 100,
    num_discrete_steps: int = 10,
    beta_schedule: str = "squaredcos_cap_v2",
    coordinate_range: DLTCoordinateRange | str = DLTCoordinateRange.normalized_0_1,
) -> None:
    """Initialize DLT configuration."""
    dataset = normalize_dataset(dataset_name)
    labels = default_id2label(dataset) if id2label is None else id2label

    self.dataset_name = str(dataset)
    self.id2label = {int(key): value for key, value in labels.items()}
    self.max_num_comp = max_num_comp or max_elements_for_dataset(dataset)
    self.categories_num = categories_num or len(self.id2label) + 2

    self.latent_dim = latent_dim
    self.num_layers = num_layers
    self.num_heads = num_heads
    self.dropout_r = dropout_r
    self.activation = activation
    self.cond_emb_size = cond_emb_size
    self.cat_emb_size = cat_emb_size
    self.num_cont_timesteps = num_cont_timesteps
    self.num_discrete_steps = num_discrete_steps
    self.beta_schedule = beta_schedule
    self.coordinate_range = str(DLTCoordinateRange(coordinate_range))

normalize_dataset

normalize_dataset(
    dataset_name: DatasetName | str,
) -> DatasetName

Normalize and validate a DLT dataset name.

Parameters:

Name Type Description Default
dataset_name DatasetName | str

Shared dataset enum or public string alias.

required

Returns:

Type Description
DatasetName

Canonical shared dataset enum.

Raises:

Type Description
ValueError

If the dataset is not a supported DLT target.

Examples:

>>> str(normalize_dataset("rico13"))
'rico13'
Source code in models/dlt/src/dlt/configuration_dlt.py
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
def normalize_dataset(dataset_name: DatasetName | str) -> DatasetName:
    """Normalize and validate a DLT dataset name.

    Args:
        dataset_name: Shared dataset enum or public string alias.

    Returns:
        Canonical shared dataset enum.

    Raises:
        ValueError: If the dataset is not a supported DLT target.

    Examples:
        >>> str(normalize_dataset("rico13"))
        'rico13'
    """
    dataset = normalize_dataset_name(dataset_name)
    if dataset not in SUPPORTED_DATASETS:
        raise ValueError(f"Unsupported DLT dataset_name: {dataset_name}")

    return dataset

default_id2label

default_id2label(
    dataset_name: DatasetName | str,
) -> dict[int, str]

Return DLT public labels for a dataset.

Source code in models/dlt/src/dlt/configuration_dlt.py
51
52
53
def default_id2label(dataset_name: DatasetName | str) -> dict[int, str]:
    """Return DLT public labels for a dataset."""
    return id2label_for_dataset(normalize_dataset(dataset_name))

conversion

Checkpoint conversion helpers for DLT.

build_pipeline

build_pipeline(config: DLTConfig) -> DLTPipeline

Build a randomly initialized DLT pipeline from a config.

Parameters:

Name Type Description Default
config DLTConfig

DLT configuration.

required

Returns:

Type Description
DLTPipeline

Pipeline with model, scheduler, and processor components.

Source code in models/dlt/src/dlt/conversion.py
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
def build_pipeline(config: DLTConfig) -> DLTPipeline:
    """Build a randomly initialized DLT pipeline from a config.

    Args:
        config: DLT configuration.

    Returns:
        Pipeline with model, scheduler, and processor components.
    """
    model = DLT(
        categories_num=config.categories_num,
        latent_dim=config.latent_dim,
        num_layers=config.num_layers,
        num_heads=config.num_heads,
        dropout_r=config.dropout_r,
        activation=config.activation,
        cond_emb_size=config.cond_emb_size,
        cat_emb_size=config.cat_emb_size,
    )
    scheduler = DLTJointDiffusionScheduler(
        alpha=0.0,
        seq_max_length=config.max_num_comp,
        discrete_features_names=[("cat", config.categories_num)],
        num_discrete_steps=[config.num_discrete_steps],
        num_train_timesteps=config.num_cont_timesteps,
        beta_schedule=config.beta_schedule,
        prediction_type="sample",
        clip_sample=False,
    )
    processor = DLTProcessor(
        dataset=config.dataset_name,
        labels=tuple(config.id2label.values()),
        max_num_comp=config.max_num_comp,
    )
    return DLTPipeline(
        model=model, scheduler=scheduler, config=config, processor=processor
    )

convert_save_pretrained_directory

convert_save_pretrained_directory(
    checkpoint_dir: str | Path,
    output_dir: str | Path,
    *,
    config: DLTConfig,
) -> DLTPipeline

Convert an original DLT save_pretrained directory into a pipeline.

Parameters:

Name Type Description Default
checkpoint_dir str | Path

Directory containing the original DLT model files.

required
output_dir str | Path

Destination directory for the converted pipeline.

required
config DLTConfig

Dataset and scheduler metadata for the checkpoint.

required

Returns:

Type Description
DLTPipeline

The saved converted pipeline.

Raises:

Type Description
RuntimeError

If the checkpoint does not match the configured model.

Source code in models/dlt/src/dlt/conversion.py
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
def convert_save_pretrained_directory(
    checkpoint_dir: str | Path,
    output_dir: str | Path,
    *,
    config: DLTConfig,
) -> DLTPipeline:
    """Convert an original DLT ``save_pretrained`` directory into a pipeline.

    Args:
        checkpoint_dir: Directory containing the original DLT model files.
        output_dir: Destination directory for the converted pipeline.
        config: Dataset and scheduler metadata for the checkpoint.

    Returns:
        The saved converted pipeline.

    Raises:
        RuntimeError: If the checkpoint does not match the configured model.
    """
    pipe = build_pipeline(config)
    loaded_model = DLT.from_pretrained(checkpoint_dir)
    missing, unexpected = pipe.model.load_state_dict(
        loaded_model.state_dict(), strict=True
    )
    if missing or unexpected:
        raise RuntimeError(f"Missing keys: {missing}; unexpected keys: {unexpected}")

    pipe.save_pretrained(output_dir)
    return pipe

modeling_dlt

DLT transformer denoiser with checkpoint state-dict key compatibility.

DLTModelOutput dataclass

Bases: BaseOutput

Output returned by the DLT denoiser.

Attributes:

Name Type Description
box Float[Tensor, 'batch elements 4']

Predicted clean internal-range boxes.

logits Float[Tensor, 'batch elements categories']

Category logits.

Source code in models/dlt/src/dlt/modeling_dlt.py
19
20
21
22
23
24
25
26
27
28
29
@dataclass
class DLTModelOutput(BaseOutput):
    """Output returned by the DLT denoiser.

    Attributes:
        box: Predicted clean internal-range boxes.
        logits: Category logits.
    """

    box: Float[torch.Tensor, "batch elements 4"]
    logits: Float[torch.Tensor, "batch elements categories"]

PositionalEncoding

Bases: Module

Sinusoidal positional encoding used by the DLT denoiser.

Source code in models/dlt/src/dlt/modeling_dlt.py
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
class PositionalEncoding(nn.Module):
    """Sinusoidal positional encoding used by the DLT denoiser."""

    pe: Float[torch.Tensor, "max_len 1 d_model"]

    def __init__(
        self, d_model: int, dropout: float = 0.05, max_len: int = 5000
    ) -> None:
        """Create the sinusoidal encoding table."""
        super().__init__()
        self.dropout = nn.Dropout(p=dropout)
        pe = torch.zeros(max_len, d_model)
        position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)
        div_term = torch.exp(
            torch.arange(0, d_model, 2).float() * (-np.log(10000.0) / d_model)
        )
        pe[:, 0::2] = torch.sin(position * div_term)
        pe[:, 1::2] = torch.cos(position * div_term)
        pe = pe.unsqueeze(0).transpose(0, 1)
        self.register_buffer("pe", pe)

    def forward(
        self, x: Float[torch.Tensor, "sequence batch channels"]
    ) -> Float[torch.Tensor, "sequence batch channels"]:
        """Add positional encodings to a sequence tensor."""
        x = x + self.pe[: x.shape[0], :]
        return self.dropout(x)

__init__

__init__(
    d_model: int, dropout: float = 0.05, max_len: int = 5000
) -> None

Create the sinusoidal encoding table.

Source code in models/dlt/src/dlt/modeling_dlt.py
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
def __init__(
    self, d_model: int, dropout: float = 0.05, max_len: int = 5000
) -> None:
    """Create the sinusoidal encoding table."""
    super().__init__()
    self.dropout = nn.Dropout(p=dropout)
    pe = torch.zeros(max_len, d_model)
    position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)
    div_term = torch.exp(
        torch.arange(0, d_model, 2).float() * (-np.log(10000.0) / d_model)
    )
    pe[:, 0::2] = torch.sin(position * div_term)
    pe[:, 1::2] = torch.cos(position * div_term)
    pe = pe.unsqueeze(0).transpose(0, 1)
    self.register_buffer("pe", pe)

forward

forward(
    x: Float[Tensor, "sequence batch channels"],
) -> Float[torch.Tensor, "sequence batch channels"]

Add positional encodings to a sequence tensor.

Source code in models/dlt/src/dlt/modeling_dlt.py
53
54
55
56
57
58
def forward(
    self, x: Float[torch.Tensor, "sequence batch channels"]
) -> Float[torch.Tensor, "sequence batch channels"]:
    """Add positional encodings to a sequence tensor."""
    x = x + self.pe[: x.shape[0], :]
    return self.dropout(x)

TimestepEmbedder

Bases: Module

Timestep MLP used by the DLT denoiser.

Source code in models/dlt/src/dlt/modeling_dlt.py
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
class TimestepEmbedder(nn.Module):
    """Timestep MLP used by the DLT denoiser."""

    def __init__(self, latent_dim: int, seq_pos_enc: PositionalEncoding) -> None:
        """Initialize the timestep embedder."""
        super().__init__()
        self.seq_pos_enc = seq_pos_enc
        self.time_embed = nn.Sequential(
            nn.Linear(latent_dim, latent_dim),
            nn.SiLU(),
            nn.Linear(latent_dim, latent_dim),
        )

    def forward(
        self, timesteps: Int[torch.Tensor, "batch"]
    ) -> Float[torch.Tensor, "1 batch channels"]:
        """Embed diffusion timesteps."""
        return self.time_embed(self.seq_pos_enc.pe[timesteps]).permute(1, 0, 2)

__init__

__init__(
    latent_dim: int, seq_pos_enc: PositionalEncoding
) -> None

Initialize the timestep embedder.

Source code in models/dlt/src/dlt/modeling_dlt.py
64
65
66
67
68
69
70
71
72
def __init__(self, latent_dim: int, seq_pos_enc: PositionalEncoding) -> None:
    """Initialize the timestep embedder."""
    super().__init__()
    self.seq_pos_enc = seq_pos_enc
    self.time_embed = nn.Sequential(
        nn.Linear(latent_dim, latent_dim),
        nn.SiLU(),
        nn.Linear(latent_dim, latent_dim),
    )

forward

forward(
    timesteps: Int[Tensor, "batch"],
) -> Float[torch.Tensor, "1 batch channels"]

Embed diffusion timesteps.

Source code in models/dlt/src/dlt/modeling_dlt.py
74
75
76
77
78
def forward(
    self, timesteps: Int[torch.Tensor, "batch"]
) -> Float[torch.Tensor, "1 batch channels"]:
    """Embed diffusion timesteps."""
    return self.time_embed(self.seq_pos_enc.pe[timesteps]).permute(1, 0, 2)

DLT

Bases: ModelMixin, ConfigMixin

Joint continuous/discrete DLT denoiser.

The module names intentionally match released checkpoint keys so model.save_pretrained directories can load without key rewriting.

Parameters:

Name Type Description Default
categories_num int

Internal category count including pad and mask/drop ids.

required
latent_dim int

Transformer latent dimension.

256
num_layers int

Number of transformer encoder layers.

4
num_heads int

Number of attention heads.

4
dropout_r float

Dropout probability.

0.0
activation str

Transformer activation.

'gelu'
cond_emb_size int

Box-condition embedding size.

224
cat_emb_size int

Category embedding size.

64
Source code in models/dlt/src/dlt/modeling_dlt.py
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
class DLT(ModelMixin, ConfigMixin):
    """Joint continuous/discrete DLT denoiser.

    The module names intentionally match released checkpoint keys so
    ``model.save_pretrained`` directories can load without key rewriting.

    Args:
        categories_num: Internal category count including pad and mask/drop ids.
        latent_dim: Transformer latent dimension.
        num_layers: Number of transformer encoder layers.
        num_heads: Number of attention heads.
        dropout_r: Dropout probability.
        activation: Transformer activation.
        cond_emb_size: Box-condition embedding size.
        cat_emb_size: Category embedding size.
    """

    config_name = "model_config.json"

    @register_to_config
    def __init__(
        self,
        categories_num: int,
        latent_dim: int = 256,
        num_layers: int = 4,
        num_heads: int = 4,
        dropout_r: float = 0.0,
        activation: str = "gelu",
        cond_emb_size: int = 224,
        cat_emb_size: int = 64,
    ) -> None:
        """Initialize the DLT denoiser."""
        super().__init__()
        self.latent_dim = latent_dim
        self.dropout_r = dropout_r
        self.categories_num = categories_num
        self.seq_pos_enc = PositionalEncoding(self.latent_dim, self.dropout_r)
        self.cat_emb = nn.Parameter(torch.randn(self.categories_num, cat_emb_size))
        self.cond_mask_box_emb = nn.Parameter(torch.randn(2, cond_emb_size))
        self.cond_mask_cat_emb = nn.Parameter(torch.randn(2, cat_emb_size))

        seq_trans_encoder_layer = nn.TransformerEncoderLayer(
            d_model=self.latent_dim,
            nhead=num_heads,
            dim_feedforward=self.latent_dim * 2,
            dropout=dropout_r,
            activation=activation,
        )
        self.seqTransEncoder = nn.TransformerEncoder(
            seq_trans_encoder_layer, num_layers=num_layers
        )
        self.embed_timestep = TimestepEmbedder(self.latent_dim, self.seq_pos_enc)
        self.output_process = nn.Sequential(nn.Linear(self.latent_dim, 4))
        self.output_cls = nn.Sequential(nn.Linear(self.latent_dim, categories_num))
        self.size_emb = nn.Sequential(nn.Linear(2, cond_emb_size))
        self.loc_emb = nn.Sequential(nn.Linear(2, cond_emb_size))

    def forward(
        self,
        sample: dict[
            str,
            Float[torch.Tensor, "batch elements channels"]
            | Int[torch.Tensor, "batch elements"],
        ],
        noisy_sample: dict[
            str,
            Float[torch.Tensor, "batch elements channels"]
            | Int[torch.Tensor, "batch elements"],
        ],
        timesteps: Int[torch.Tensor, "batch"],
        return_dict: bool = False,
    ) -> (
        DLTModelOutput
        | tuple[
            Float[torch.Tensor, "batch elements 4"],
            Float[torch.Tensor, "batch elements categories"],
        ]
    ):
        """Predict clean boxes and category logits for a noisy layout.

        Args:
            sample: DLT-format conditioning batch with ``box_cond``,
                ``cat``, ``mask_box``, and ``mask_cat``.
            noisy_sample: Current noisy ``box`` and ``cat`` tensors.
            timesteps: Continuous diffusion timestep per batch item.
            return_dict: Whether to return ``DLTModelOutput``.

        Returns:
            Either a two-tuple ``(box, logits)`` for checkpoint-compatible
            callers or a dataclass output.
        """
        cat_input = (
            noisy_sample["cat"] * sample["mask_cat"]
            + (1 - sample["mask_cat"]) * sample["cat"]
        )
        cat_input_flat = rearrange(cat_input, "b c -> (b c)")
        sample_tensor = (
            sample["mask_box"] * noisy_sample["box"]
            + (1 - sample["mask_box"]) * sample["box_cond"]
        )

        xy = sample_tensor[:, :, :2]
        wh = sample_tensor[:, :, 2:]

        elem_cat_emb = self.cat_emb[cat_input_flat, :]
        elem_cat_emb = rearrange(
            elem_cat_emb, "(b c) d -> b c d", b=noisy_sample["box"].shape[0]
        )

        def mask_to_emb(
            mask: Int[torch.Tensor, "batch elements"],
            cond_mask_emb: Float[torch.Tensor, "mask channels"],
        ) -> Float[torch.Tensor, "batch elements channels"]:
            mask_flat = rearrange(mask, "b c -> (b c)").long()
            mask_all_emb = cond_mask_emb[mask_flat, :]
            return rearrange(mask_all_emb, "(b c) d -> b c d", b=mask.shape[0])

        emb_mask_wh = mask_to_emb(sample["mask_box"][:, :, 2], self.cond_mask_box_emb)
        emb_mask_xy = mask_to_emb(sample["mask_box"][:, :, 0], self.cond_mask_box_emb)
        emb_mask_cl = mask_to_emb(sample["mask_cat"], self.cond_mask_cat_emb)
        t_emb = self.embed_timestep(timesteps)

        size_emb = self.size_emb(wh) + emb_mask_wh
        loc_emb = self.loc_emb(xy) + emb_mask_xy
        elem_cat_emb = elem_cat_emb + emb_mask_cl

        tokens_emb = torch.cat([size_emb, loc_emb, elem_cat_emb], dim=-1)
        tokens_emb = rearrange(tokens_emb, "b c d -> c b d")
        xseq = torch.cat((t_emb, tokens_emb), dim=0)
        xseq = self.seq_pos_enc(xseq)

        output = self.seqTransEncoder(xseq)[1:]
        output = rearrange(output, "c b d -> b c d")
        output_box = self.output_process(output)
        output_cls = self.output_cls(output)
        if not return_dict:
            return output_box, output_cls
        return DLTModelOutput(box=output_box, logits=output_cls)

    def save_pretrained(
        self,
        save_directory: str | os.PathLike[str],
        is_main_process: bool = True,
        save_function: Callable[..., None] | None = None,
        safe_serialization: bool = False,
        variant: str | None = None,
        max_shard_size: int | str = "10GB",
        push_to_hub: bool = False,
        use_flashpack: bool = False,
        **kwargs: str | int | bool | float | None,
    ) -> None:
        """Save the model with PyTorch serialization by default.

        DLT keeps shared positional-encoding buffers that safetensors refuses
        to flatten.
        """
        super().save_pretrained(
            save_directory,
            is_main_process=is_main_process,
            save_function=save_function,
            safe_serialization=safe_serialization,
            variant=variant,
            max_shard_size=max_shard_size,
            push_to_hub=push_to_hub,
            use_flashpack=use_flashpack,
            **kwargs,
        )

__init__

__init__(
    categories_num: int,
    latent_dim: int = 256,
    num_layers: int = 4,
    num_heads: int = 4,
    dropout_r: float = 0.0,
    activation: str = "gelu",
    cond_emb_size: int = 224,
    cat_emb_size: int = 64,
) -> None

Initialize the DLT denoiser.

Source code in models/dlt/src/dlt/modeling_dlt.py
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
@register_to_config
def __init__(
    self,
    categories_num: int,
    latent_dim: int = 256,
    num_layers: int = 4,
    num_heads: int = 4,
    dropout_r: float = 0.0,
    activation: str = "gelu",
    cond_emb_size: int = 224,
    cat_emb_size: int = 64,
) -> None:
    """Initialize the DLT denoiser."""
    super().__init__()
    self.latent_dim = latent_dim
    self.dropout_r = dropout_r
    self.categories_num = categories_num
    self.seq_pos_enc = PositionalEncoding(self.latent_dim, self.dropout_r)
    self.cat_emb = nn.Parameter(torch.randn(self.categories_num, cat_emb_size))
    self.cond_mask_box_emb = nn.Parameter(torch.randn(2, cond_emb_size))
    self.cond_mask_cat_emb = nn.Parameter(torch.randn(2, cat_emb_size))

    seq_trans_encoder_layer = nn.TransformerEncoderLayer(
        d_model=self.latent_dim,
        nhead=num_heads,
        dim_feedforward=self.latent_dim * 2,
        dropout=dropout_r,
        activation=activation,
    )
    self.seqTransEncoder = nn.TransformerEncoder(
        seq_trans_encoder_layer, num_layers=num_layers
    )
    self.embed_timestep = TimestepEmbedder(self.latent_dim, self.seq_pos_enc)
    self.output_process = nn.Sequential(nn.Linear(self.latent_dim, 4))
    self.output_cls = nn.Sequential(nn.Linear(self.latent_dim, categories_num))
    self.size_emb = nn.Sequential(nn.Linear(2, cond_emb_size))
    self.loc_emb = nn.Sequential(nn.Linear(2, cond_emb_size))

forward

forward(
    sample: dict[
        str,
        Float[Tensor, "batch elements channels"]
        | Int[Tensor, "batch elements"],
    ],
    noisy_sample: dict[
        str,
        Float[Tensor, "batch elements channels"]
        | Int[Tensor, "batch elements"],
    ],
    timesteps: Int[Tensor, "batch"],
    return_dict: bool = False,
) -> (
    DLTModelOutput
    | tuple[
        Float[torch.Tensor, "batch elements 4"],
        Float[torch.Tensor, "batch elements categories"],
    ]
)

Predict clean boxes and category logits for a noisy layout.

Parameters:

Name Type Description Default
sample dict[str, Float[Tensor, 'batch elements channels'] | Int[Tensor, 'batch elements']]

DLT-format conditioning batch with box_cond, cat, mask_box, and mask_cat.

required
noisy_sample dict[str, Float[Tensor, 'batch elements channels'] | Int[Tensor, 'batch elements']]

Current noisy box and cat tensors.

required
timesteps Int[Tensor, 'batch']

Continuous diffusion timestep per batch item.

required
return_dict bool

Whether to return DLTModelOutput.

False

Returns:

Type Description
DLTModelOutput | tuple[Float[Tensor, 'batch elements 4'], Float[Tensor, 'batch elements categories']]

Either a two-tuple (box, logits) for checkpoint-compatible

DLTModelOutput | tuple[Float[Tensor, 'batch elements 4'], Float[Tensor, 'batch elements categories']]

callers or a dataclass output.

Source code in models/dlt/src/dlt/modeling_dlt.py
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
def forward(
    self,
    sample: dict[
        str,
        Float[torch.Tensor, "batch elements channels"]
        | Int[torch.Tensor, "batch elements"],
    ],
    noisy_sample: dict[
        str,
        Float[torch.Tensor, "batch elements channels"]
        | Int[torch.Tensor, "batch elements"],
    ],
    timesteps: Int[torch.Tensor, "batch"],
    return_dict: bool = False,
) -> (
    DLTModelOutput
    | tuple[
        Float[torch.Tensor, "batch elements 4"],
        Float[torch.Tensor, "batch elements categories"],
    ]
):
    """Predict clean boxes and category logits for a noisy layout.

    Args:
        sample: DLT-format conditioning batch with ``box_cond``,
            ``cat``, ``mask_box``, and ``mask_cat``.
        noisy_sample: Current noisy ``box`` and ``cat`` tensors.
        timesteps: Continuous diffusion timestep per batch item.
        return_dict: Whether to return ``DLTModelOutput``.

    Returns:
        Either a two-tuple ``(box, logits)`` for checkpoint-compatible
        callers or a dataclass output.
    """
    cat_input = (
        noisy_sample["cat"] * sample["mask_cat"]
        + (1 - sample["mask_cat"]) * sample["cat"]
    )
    cat_input_flat = rearrange(cat_input, "b c -> (b c)")
    sample_tensor = (
        sample["mask_box"] * noisy_sample["box"]
        + (1 - sample["mask_box"]) * sample["box_cond"]
    )

    xy = sample_tensor[:, :, :2]
    wh = sample_tensor[:, :, 2:]

    elem_cat_emb = self.cat_emb[cat_input_flat, :]
    elem_cat_emb = rearrange(
        elem_cat_emb, "(b c) d -> b c d", b=noisy_sample["box"].shape[0]
    )

    def mask_to_emb(
        mask: Int[torch.Tensor, "batch elements"],
        cond_mask_emb: Float[torch.Tensor, "mask channels"],
    ) -> Float[torch.Tensor, "batch elements channels"]:
        mask_flat = rearrange(mask, "b c -> (b c)").long()
        mask_all_emb = cond_mask_emb[mask_flat, :]
        return rearrange(mask_all_emb, "(b c) d -> b c d", b=mask.shape[0])

    emb_mask_wh = mask_to_emb(sample["mask_box"][:, :, 2], self.cond_mask_box_emb)
    emb_mask_xy = mask_to_emb(sample["mask_box"][:, :, 0], self.cond_mask_box_emb)
    emb_mask_cl = mask_to_emb(sample["mask_cat"], self.cond_mask_cat_emb)
    t_emb = self.embed_timestep(timesteps)

    size_emb = self.size_emb(wh) + emb_mask_wh
    loc_emb = self.loc_emb(xy) + emb_mask_xy
    elem_cat_emb = elem_cat_emb + emb_mask_cl

    tokens_emb = torch.cat([size_emb, loc_emb, elem_cat_emb], dim=-1)
    tokens_emb = rearrange(tokens_emb, "b c d -> c b d")
    xseq = torch.cat((t_emb, tokens_emb), dim=0)
    xseq = self.seq_pos_enc(xseq)

    output = self.seqTransEncoder(xseq)[1:]
    output = rearrange(output, "c b d -> b c d")
    output_box = self.output_process(output)
    output_cls = self.output_cls(output)
    if not return_dict:
        return output_box, output_cls
    return DLTModelOutput(box=output_box, logits=output_cls)

save_pretrained

save_pretrained(
    save_directory: str | PathLike[str],
    is_main_process: bool = True,
    save_function: Callable[..., None] | None = None,
    safe_serialization: bool = False,
    variant: str | None = None,
    max_shard_size: int | str = "10GB",
    push_to_hub: bool = False,
    use_flashpack: bool = False,
    **kwargs: str | int | bool | float | None,
) -> None

Save the model with PyTorch serialization by default.

DLT keeps shared positional-encoding buffers that safetensors refuses to flatten.

Source code in models/dlt/src/dlt/modeling_dlt.py
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
def save_pretrained(
    self,
    save_directory: str | os.PathLike[str],
    is_main_process: bool = True,
    save_function: Callable[..., None] | None = None,
    safe_serialization: bool = False,
    variant: str | None = None,
    max_shard_size: int | str = "10GB",
    push_to_hub: bool = False,
    use_flashpack: bool = False,
    **kwargs: str | int | bool | float | None,
) -> None:
    """Save the model with PyTorch serialization by default.

    DLT keeps shared positional-encoding buffers that safetensors refuses
    to flatten.
    """
    super().save_pretrained(
        save_directory,
        is_main_process=is_main_process,
        save_function=save_function,
        safe_serialization=safe_serialization,
        variant=variant,
        max_shard_size=max_shard_size,
        push_to_hub=push_to_hub,
        use_flashpack=use_flashpack,
        **kwargs,
    )

pipeline_dlt

Diffusers pipeline for DLT layout generation.

OutputType

Bases: StrEnum

DLT pipeline output containers.

Source code in models/dlt/src/dlt/pipeline_dlt.py
21
22
23
24
25
class OutputType(StrEnum):
    """DLT pipeline output containers."""

    dataclass = auto()
    dict = auto()

DLTConditionAlias

Bases: StrEnum

DLT checkpoint condition aliases.

Source code in models/dlt/src/dlt/pipeline_dlt.py
28
29
30
31
32
33
class DLTConditionAlias(StrEnum):
    """DLT checkpoint condition aliases."""

    all = auto()
    whole_box = auto()
    loc = auto()

DLTPipeline

Bases: DiffusionPipeline

Generate layouts with a converted DLT checkpoint.

Parameters:

Name Type Description Default
model DLT

DLT denoiser.

required
scheduler DLTJointDiffusionScheduler

Joint box/category scheduler.

required
config DLTConfig

Pipeline configuration.

required
processor DLTProcessor | None

Layout processor.

None
Source code in models/dlt/src/dlt/pipeline_dlt.py
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
class DLTPipeline(DiffusionPipeline):
    """Generate layouts with a converted DLT checkpoint.

    Args:
        model: DLT denoiser.
        scheduler: Joint box/category scheduler.
        config: Pipeline configuration.
        processor: Layout processor.
    """

    model_cpu_offload_seq = "model"
    _optional_components = ["processor"]

    def __init__(
        self,
        model: DLT,
        scheduler: DLTJointDiffusionScheduler,
        config: DLTConfig,
        processor: DLTProcessor | None = None,
    ) -> None:
        """Initialize a DLT pipeline."""
        super().__init__()
        self.register_modules(model=model, scheduler=scheduler)
        self.dlt_config = config
        self.processor = processor or DLTProcessor(
            dataset=self.dlt_config.dataset_name,
            labels=tuple(self.dlt_config.id2label.values()),
            max_num_comp=self.dlt_config.max_num_comp,
        )
        self.model.eval()

    @property
    def components(self) -> PipelineComponents:
        """Return serializable pipeline components."""
        return {
            "model": self.model,
            "scheduler": self.scheduler,
            "processor": self.processor,
        }

    @torch.no_grad()
    def __call__(
        self,
        *,
        batch_size: int = 1,
        seed: int | None = None,
        generator: torch.Generator | None = None,
        condition_type: ConditionType | str | None = ConditionType.unconditional,
        labels: Int[torch.Tensor, "batch elements"] | None = None,
        bbox: Float[torch.Tensor, "batch elements 4"] | None = None,
        mask: Bool[torch.Tensor, "batch elements"] | None = None,
        num_elements: int | list[int] | Int[torch.Tensor, "batch"] | None = None,
        box_format: BoxFormat | str = BoxFormat.xywh,
        normalized: bool = True,
        canvas_size: tuple[int, int] | None = None,
        num_inference_steps: int | None = None,
        temperature: float | None = None,
        output_type: OutputType | str = OutputType.dataclass,
        return_intermediates: bool = False,
    ) -> (
        LayoutGenerationOutput
        | dict[
            str,
            Float[torch.Tensor, "..."]
            | Int[torch.Tensor, "..."]
            | Bool[torch.Tensor, "..."]
            | dict[int, str]
            | list[Float[torch.Tensor, "..."]]
            | dict[str, str]
            | None,
        ]
    ):
        """Run DLT joint denoising and return generated layouts.

        Args:
            batch_size: Number of layouts to generate.
            seed: Optional seed used when ``generator`` is absent.
            generator: Optional torch generator. Takes precedence over ``seed``.
            condition_type: Canonical condition or DLT checkpoint alias.
            labels: Optional public labels for conditioned modes.
            bbox: Optional public boxes for conditioned modes.
            mask: Optional valid-element mask.
            num_elements: Optional valid element count for unconditional calls.
            box_format: Input box format.
            normalized: Whether input boxes are normalized.
            canvas_size: Pixel canvas size for non-normalized boxes.
            num_inference_steps: Number of reverse diffusion steps.
            temperature: Optional category sampling temperature override.
            output_type: ``"dataclass"`` or ``"dict"``.
            return_intermediates: Whether to include denoising trajectory.

        Returns:
            Layout generation output dataclass or dictionary.

        Raises:
            ValueError: If the condition or output type is unsupported.
        """
        canonical = normalize_condition_type(condition_type)
        output_kind = OutputType(output_type)
        if generator is None and seed is not None:
            generator = torch.Generator(device=self.device).manual_seed(seed)
        if canonical is ConditionType.unconditional:
            processed = self.processor.empty_condition(
                batch_size=batch_size, device=self.device
            )
            if num_elements is not None:
                lengths = torch.as_tensor(
                    num_elements, dtype=torch.long, device=self.device
                )
                if lengths.ndim == 0:
                    lengths = lengths.repeat(batch_size)
                processed["mask"] = (
                    torch.arange(self.processor.max_num_comp, device=self.device)[
                        None, :
                    ]
                    < lengths[:, None]
                )
        else:
            _require_condition_inputs(
                condition_type=condition_type, bbox=bbox, labels=labels
            )
            processed = self.processor(
                bbox=bbox,
                labels=labels,
                mask=mask,
                box_format=box_format,
                normalized=normalized,
                canvas_size=canvas_size,
                device=self.device,
            )
            batch_size = processed["box"].shape[0]
        mask_box, mask_cat = self.processor.condition_masks(
            str(canonical), mask=processed["mask"]
        )
        sample = {
            "box": processed["box"],
            "box_cond": processed["box_cond"],
            "cat": processed["cat"],
            "mask_box": mask_box,
            "mask_cat": mask_cat,
        }
        noisy_batch = {
            "box": torch.randn(
                processed["box"].shape,
                dtype=processed["box"].dtype,
                device=self.device,
                generator=generator,
            ),
            "cat": torch.full(
                processed["cat"].shape,
                self.processor.mask_category_id,
                dtype=torch.long,
                device=self.device,
            ),
        }
        original_temperature = self.scheduler.temperature
        if temperature is not None:
            self.scheduler.temperature = temperature
        steps = max(1, num_inference_steps or self.scheduler.num_train_timesteps)
        trajectory: list[Float[torch.Tensor, "batch elements 4"]] | None = (
            [] if return_intermediates else None
        )
        bbox_step: DLTJointSchedulerOutput | None = None
        try:
            for i in range(steps - 1, -1, -1):
                t_value = min(i, self.scheduler.num_train_timesteps - 1)
                t = torch.tensor([t_value] * batch_size, device=self.device)
                bbox_pred, cat_pred = self.model(sample, noisy_batch, timesteps=t)
                bbox_step, cat_step = self.scheduler.step_jointly(
                    bbox_pred,
                    {"cat": cat_pred},
                    timestep=t,
                    sample=noisy_batch["box"],
                    generator=generator,
                )
                noisy_batch["box"] = bbox_step.prev_sample
                noisy_batch["cat"] = cat_step["cat"]
                if trajectory is not None:
                    trajectory.append(noisy_batch["box"].detach().cpu())
        finally:
            self.scheduler.temperature = original_temperature
        if bbox_step is None:
            raise RuntimeError("DLT denoising did not run any scheduler steps")

        final_box = (
            sample["mask_box"] * bbox_step.pred_original_sample
            + (1 - sample["mask_box"]) * sample["box_cond"]
        )
        final_cat = (
            sample["mask_cat"] * noisy_batch["cat"]
            + (1 - sample["mask_cat"]) * sample["cat"]
        )
        valid_mask = processed["mask"].detach().cpu()
        output = LayoutGenerationOutput(
            bbox=self.processor.internal_to_public_boxes(final_box).detach().cpu()
            * valid_mask.unsqueeze(-1),
            labels=self.processor.internal_to_public_labels(
                final_cat, processed["mask"]
            )
            .detach()
            .cpu(),
            mask=valid_mask,
            id2label=self.processor.id2label,
            trajectory=trajectory,
            intermediates={"condition_type": str(canonical)}
            if return_intermediates
            else None,
        )
        return _format_pipeline_output(output, output_kind)

    generate = __call__

    def save_pretrained(self, save_directory: str | Path) -> None:
        """Persist DLT model, scheduler, and pipeline metadata."""
        super().save_pretrained(save_directory, safe_serialization=False)
        self.dlt_config.save_config(save_directory)

    @classmethod
    def from_pretrained(cls, pretrained_model_name_or_path: str | Path) -> Self:
        """Load a saved DLT pipeline."""
        config_dict, _ = DLTConfig.load_config(
            pretrained_model_name_or_path, return_unused_kwargs=True
        )
        config = cast(DLTConfig, DLTConfig.from_config(config_dict))
        pipe = super().from_pretrained(pretrained_model_name_or_path, config=config)
        pipe.dlt_config = config
        pipe.processor = DLTProcessor(
            dataset=config.dataset_name,
            labels=tuple(config.id2label.values()),
            max_num_comp=config.max_num_comp,
        )
        return pipe

components property

components: PipelineComponents

Return serializable pipeline components.

__init__

__init__(
    model: DLT,
    scheduler: DLTJointDiffusionScheduler,
    config: DLTConfig,
    processor: DLTProcessor | None = None,
) -> None

Initialize a DLT pipeline.

Source code in models/dlt/src/dlt/pipeline_dlt.py
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
def __init__(
    self,
    model: DLT,
    scheduler: DLTJointDiffusionScheduler,
    config: DLTConfig,
    processor: DLTProcessor | None = None,
) -> None:
    """Initialize a DLT pipeline."""
    super().__init__()
    self.register_modules(model=model, scheduler=scheduler)
    self.dlt_config = config
    self.processor = processor or DLTProcessor(
        dataset=self.dlt_config.dataset_name,
        labels=tuple(self.dlt_config.id2label.values()),
        max_num_comp=self.dlt_config.max_num_comp,
    )
    self.model.eval()

__call__

__call__(
    *,
    batch_size: int = 1,
    seed: int | None = None,
    generator: Generator | None = None,
    condition_type: ConditionType
    | str
    | None = ConditionType.unconditional,
    labels: Int[Tensor, "batch elements"] | None = None,
    bbox: Float[Tensor, "batch elements 4"] | None = None,
    mask: Bool[Tensor, "batch elements"] | None = None,
    num_elements: int
    | list[int]
    | Int[Tensor, batch]
    | None = None,
    box_format: BoxFormat | str = BoxFormat.xywh,
    normalized: bool = True,
    canvas_size: tuple[int, int] | None = None,
    num_inference_steps: int | None = None,
    temperature: float | None = None,
    output_type: OutputType | str = OutputType.dataclass,
    return_intermediates: bool = False,
) -> (
    LayoutGenerationOutput
    | dict[
        str,
        Float[torch.Tensor, ...]
        | Int[torch.Tensor, ...]
        | Bool[torch.Tensor, ...]
        | dict[int, str]
        | list[Float[torch.Tensor, ...]]
        | dict[str, str]
        | None,
    ]
)

Run DLT joint denoising and return generated layouts.

Parameters:

Name Type Description Default
batch_size int

Number of layouts to generate.

1
seed int | None

Optional seed used when generator is absent.

None
generator Generator | None

Optional torch generator. Takes precedence over seed.

None
condition_type ConditionType | str | None

Canonical condition or DLT checkpoint alias.

unconditional
labels Int[Tensor, 'batch elements'] | None

Optional public labels for conditioned modes.

None
bbox Float[Tensor, 'batch elements 4'] | None

Optional public boxes for conditioned modes.

None
mask Bool[Tensor, 'batch elements'] | None

Optional valid-element mask.

None
num_elements int | list[int] | Int[Tensor, batch] | None

Optional valid element count for unconditional calls.

None
box_format BoxFormat | str

Input box format.

xywh
normalized bool

Whether input boxes are normalized.

True
canvas_size tuple[int, int] | None

Pixel canvas size for non-normalized boxes.

None
num_inference_steps int | None

Number of reverse diffusion steps.

None
temperature float | None

Optional category sampling temperature override.

None
output_type OutputType | str

"dataclass" or "dict".

dataclass
return_intermediates bool

Whether to include denoising trajectory.

False

Returns:

Type Description
LayoutGenerationOutput | dict[str, Float[Tensor, ...] | Int[Tensor, ...] | Bool[Tensor, ...] | dict[int, str] | list[Float[Tensor, ...]] | dict[str, str] | None]

Layout generation output dataclass or dictionary.

Raises:

Type Description
ValueError

If the condition or output type is unsupported.

Source code in models/dlt/src/dlt/pipeline_dlt.py
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
@torch.no_grad()
def __call__(
    self,
    *,
    batch_size: int = 1,
    seed: int | None = None,
    generator: torch.Generator | None = None,
    condition_type: ConditionType | str | None = ConditionType.unconditional,
    labels: Int[torch.Tensor, "batch elements"] | None = None,
    bbox: Float[torch.Tensor, "batch elements 4"] | None = None,
    mask: Bool[torch.Tensor, "batch elements"] | None = None,
    num_elements: int | list[int] | Int[torch.Tensor, "batch"] | None = None,
    box_format: BoxFormat | str = BoxFormat.xywh,
    normalized: bool = True,
    canvas_size: tuple[int, int] | None = None,
    num_inference_steps: int | None = None,
    temperature: float | None = None,
    output_type: OutputType | str = OutputType.dataclass,
    return_intermediates: bool = False,
) -> (
    LayoutGenerationOutput
    | dict[
        str,
        Float[torch.Tensor, "..."]
        | Int[torch.Tensor, "..."]
        | Bool[torch.Tensor, "..."]
        | dict[int, str]
        | list[Float[torch.Tensor, "..."]]
        | dict[str, str]
        | None,
    ]
):
    """Run DLT joint denoising and return generated layouts.

    Args:
        batch_size: Number of layouts to generate.
        seed: Optional seed used when ``generator`` is absent.
        generator: Optional torch generator. Takes precedence over ``seed``.
        condition_type: Canonical condition or DLT checkpoint alias.
        labels: Optional public labels for conditioned modes.
        bbox: Optional public boxes for conditioned modes.
        mask: Optional valid-element mask.
        num_elements: Optional valid element count for unconditional calls.
        box_format: Input box format.
        normalized: Whether input boxes are normalized.
        canvas_size: Pixel canvas size for non-normalized boxes.
        num_inference_steps: Number of reverse diffusion steps.
        temperature: Optional category sampling temperature override.
        output_type: ``"dataclass"`` or ``"dict"``.
        return_intermediates: Whether to include denoising trajectory.

    Returns:
        Layout generation output dataclass or dictionary.

    Raises:
        ValueError: If the condition or output type is unsupported.
    """
    canonical = normalize_condition_type(condition_type)
    output_kind = OutputType(output_type)
    if generator is None and seed is not None:
        generator = torch.Generator(device=self.device).manual_seed(seed)
    if canonical is ConditionType.unconditional:
        processed = self.processor.empty_condition(
            batch_size=batch_size, device=self.device
        )
        if num_elements is not None:
            lengths = torch.as_tensor(
                num_elements, dtype=torch.long, device=self.device
            )
            if lengths.ndim == 0:
                lengths = lengths.repeat(batch_size)
            processed["mask"] = (
                torch.arange(self.processor.max_num_comp, device=self.device)[
                    None, :
                ]
                < lengths[:, None]
            )
    else:
        _require_condition_inputs(
            condition_type=condition_type, bbox=bbox, labels=labels
        )
        processed = self.processor(
            bbox=bbox,
            labels=labels,
            mask=mask,
            box_format=box_format,
            normalized=normalized,
            canvas_size=canvas_size,
            device=self.device,
        )
        batch_size = processed["box"].shape[0]
    mask_box, mask_cat = self.processor.condition_masks(
        str(canonical), mask=processed["mask"]
    )
    sample = {
        "box": processed["box"],
        "box_cond": processed["box_cond"],
        "cat": processed["cat"],
        "mask_box": mask_box,
        "mask_cat": mask_cat,
    }
    noisy_batch = {
        "box": torch.randn(
            processed["box"].shape,
            dtype=processed["box"].dtype,
            device=self.device,
            generator=generator,
        ),
        "cat": torch.full(
            processed["cat"].shape,
            self.processor.mask_category_id,
            dtype=torch.long,
            device=self.device,
        ),
    }
    original_temperature = self.scheduler.temperature
    if temperature is not None:
        self.scheduler.temperature = temperature
    steps = max(1, num_inference_steps or self.scheduler.num_train_timesteps)
    trajectory: list[Float[torch.Tensor, "batch elements 4"]] | None = (
        [] if return_intermediates else None
    )
    bbox_step: DLTJointSchedulerOutput | None = None
    try:
        for i in range(steps - 1, -1, -1):
            t_value = min(i, self.scheduler.num_train_timesteps - 1)
            t = torch.tensor([t_value] * batch_size, device=self.device)
            bbox_pred, cat_pred = self.model(sample, noisy_batch, timesteps=t)
            bbox_step, cat_step = self.scheduler.step_jointly(
                bbox_pred,
                {"cat": cat_pred},
                timestep=t,
                sample=noisy_batch["box"],
                generator=generator,
            )
            noisy_batch["box"] = bbox_step.prev_sample
            noisy_batch["cat"] = cat_step["cat"]
            if trajectory is not None:
                trajectory.append(noisy_batch["box"].detach().cpu())
    finally:
        self.scheduler.temperature = original_temperature
    if bbox_step is None:
        raise RuntimeError("DLT denoising did not run any scheduler steps")

    final_box = (
        sample["mask_box"] * bbox_step.pred_original_sample
        + (1 - sample["mask_box"]) * sample["box_cond"]
    )
    final_cat = (
        sample["mask_cat"] * noisy_batch["cat"]
        + (1 - sample["mask_cat"]) * sample["cat"]
    )
    valid_mask = processed["mask"].detach().cpu()
    output = LayoutGenerationOutput(
        bbox=self.processor.internal_to_public_boxes(final_box).detach().cpu()
        * valid_mask.unsqueeze(-1),
        labels=self.processor.internal_to_public_labels(
            final_cat, processed["mask"]
        )
        .detach()
        .cpu(),
        mask=valid_mask,
        id2label=self.processor.id2label,
        trajectory=trajectory,
        intermediates={"condition_type": str(canonical)}
        if return_intermediates
        else None,
    )
    return _format_pipeline_output(output, output_kind)

save_pretrained

save_pretrained(save_directory: str | Path) -> None

Persist DLT model, scheduler, and pipeline metadata.

Source code in models/dlt/src/dlt/pipeline_dlt.py
325
326
327
328
def save_pretrained(self, save_directory: str | Path) -> None:
    """Persist DLT model, scheduler, and pipeline metadata."""
    super().save_pretrained(save_directory, safe_serialization=False)
    self.dlt_config.save_config(save_directory)

from_pretrained classmethod

from_pretrained(
    pretrained_model_name_or_path: str | Path,
) -> Self

Load a saved DLT pipeline.

Source code in models/dlt/src/dlt/pipeline_dlt.py
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
@classmethod
def from_pretrained(cls, pretrained_model_name_or_path: str | Path) -> Self:
    """Load a saved DLT pipeline."""
    config_dict, _ = DLTConfig.load_config(
        pretrained_model_name_or_path, return_unused_kwargs=True
    )
    config = cast(DLTConfig, DLTConfig.from_config(config_dict))
    pipe = super().from_pretrained(pretrained_model_name_or_path, config=config)
    pipe.dlt_config = config
    pipe.processor = DLTProcessor(
        dataset=config.dataset_name,
        labels=tuple(config.id2label.values()),
        max_num_comp=config.max_num_comp,
    )
    return pipe

normalize_condition_type

normalize_condition_type(
    condition_type: ConditionType | str | None,
) -> ConditionType

Normalize public and DLT alias condition names.

Source code in models/dlt/src/dlt/pipeline_dlt.py
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
def normalize_condition_type(
    condition_type: ConditionType | str | None,
) -> ConditionType:
    """Normalize public and DLT alias condition names."""
    if condition_type is None:
        canonical = ConditionType.unconditional
    elif isinstance(condition_type, ConditionType):
        canonical = condition_type
    else:
        try:
            canonical = normalize_shared_condition_type(condition_type)
        except ValueError:
            key = condition_type.lower().replace("-", "_")
            try:
                canonical = _DLT_CONDITION_ALIASES[DLTConditionAlias(key)]
            except ValueError as exc:
                raise ValueError(
                    f"Unsupported DLT condition_type: {condition_type}"
                ) from exc

    if canonical not in _SUPPORTED_CONDITION_TYPES:
        raise ValueError(f"Unsupported DLT condition_type: {condition_type}")

    return canonical

processing_dlt

Processor for DLT public layouts and internal tensors.

DLTProcessedBatch

Bases: TypedDict

Padded DLT tensors consumed by the model and scheduler.

Source code in models/dlt/src/dlt/processing_dlt.py
23
24
25
26
27
28
29
class DLTProcessedBatch(TypedDict):
    """Padded DLT tensors consumed by the model and scheduler."""

    box: Float[torch.Tensor, "batch elements 4"]
    box_cond: Float[torch.Tensor, "batch elements 4"]
    cat: Int[torch.Tensor, "batch elements"]
    mask: Bool[torch.Tensor, "batch elements"]

DLTProcessor

Bases: ProcessorMixin

Encode DLT public inputs into the package tensor format.

Parameters:

Name Type Description Default
dataset DatasetName | str

Canonical dataset name.

required
labels Sequence[str]

Ordered public labels without internal pad/drop ids.

required
max_num_comp int

Maximum number of layout elements.

required
Source code in models/dlt/src/dlt/processing_dlt.py
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
class DLTProcessor(ProcessorMixin):
    """Encode DLT public inputs into the package tensor format.

    Args:
        dataset: Canonical dataset name.
        labels: Ordered public labels without internal pad/drop ids.
        max_num_comp: Maximum number of layout elements.
    """

    config_name = "processor_config.json"

    def __init__(
        self,
        dataset: DatasetName | str,
        labels: Sequence[str],
        max_num_comp: int,
    ) -> None:
        """Initialize processor metadata."""
        super().__init__()
        self.dataset = str(normalize_dataset(dataset))
        self.labels = tuple(str(label) for label in labels)
        self.max_num_comp = max_num_comp

    @classmethod
    def from_dataset(cls, dataset: DatasetName | str) -> "DLTProcessor":
        """Create a processor from shared dataset metadata."""
        canonical = normalize_dataset(dataset)
        return cls(
            dataset=canonical,
            labels=tuple(default_id2label(canonical).values()),
            max_num_comp=9
            if canonical in {DatasetName.publaynet, DatasetName.rico13}
            else 33,
        )

    @property
    def id2label(self) -> dict[int, str]:
        """Return public label names keyed by dataset-local ids."""
        return dict(enumerate(self.labels))

    @property
    def categories_num(self) -> int:
        """Return internal category count including pad and mask/drop ids."""
        return len(self.labels) + 2

    @property
    def pad_category_id(self) -> int:
        """Return the internal padding category id."""
        return 0

    @property
    def mask_category_id(self) -> int:
        """Return the internal mask/drop category id."""
        return self.categories_num - 1

    def __call__(
        self,
        *,
        bbox: Float[torch.Tensor, "batch elements 4"]
        | Float[np.ndarray, "batch elements 4"]
        | LayoutInput,
        labels: Int[torch.Tensor, "batch elements"]
        | Int[np.ndarray, "batch elements"]
        | LayoutInput,
        mask: Bool[torch.Tensor, "batch elements"]
        | Bool[np.ndarray, "batch elements"]
        | LayoutInput
        | None = None,
        box_format: BoxFormat | str = BoxFormat.xywh,
        normalized: bool = True,
        canvas_size: tuple[int, int] | None = None,
        device: torch.device | str | None = None,
    ) -> DLTProcessedBatch:
        """Convert public layout tensors into padded internal tensors."""
        bbox_t, labels_t, mask_t = prepare_layout_tensors(
            bbox=bbox,
            labels=labels,
            mask=mask,
            box_format=box_format,
            normalized=normalized,
            canvas_size=canvas_size,
        )
        if device is not None:
            bbox_t = bbox_t.to(device)
            labels_t = labels_t.to(device)
            mask_t = mask_t.to(device)
        bbox_t, labels_t, mask_t = self.pad(bbox_t, labels_t, mask_t)
        return cast(
            DLTProcessedBatch,
            {
                "box": self.public_to_internal_boxes(bbox_t) * mask_t.unsqueeze(-1),
                "box_cond": self.public_to_internal_boxes(bbox_t)
                * mask_t.unsqueeze(-1),
                "cat": self.public_to_internal_labels(labels_t, mask_t),
                "mask": mask_t,
            },
        )

    def empty_condition(
        self,
        *,
        batch_size: int,
        device: torch.device | str,
        dtype: torch.dtype = torch.float32,
    ) -> DLTProcessedBatch:
        """Return an empty unconditional internal batch."""
        device = torch.device(device)
        bbox = torch.zeros(batch_size, self.max_num_comp, 4, dtype=dtype, device=device)
        labels = torch.zeros(
            batch_size, self.max_num_comp, dtype=torch.long, device=device
        )
        mask = torch.ones(
            batch_size, self.max_num_comp, dtype=torch.bool, device=device
        )
        return cast(
            DLTProcessedBatch,
            {
                "box": self.public_to_internal_boxes(bbox),
                "box_cond": self.public_to_internal_boxes(bbox),
                "cat": self.public_to_internal_labels(labels, mask),
                "mask": mask,
            },
        )

    def pad(
        self,
        bbox: Float[torch.Tensor, "batch elements 4"],
        labels: Int[torch.Tensor, "batch elements"],
        mask: Bool[torch.Tensor, "batch elements"] | None = None,
    ) -> tuple[
        Float[torch.Tensor, "batch max_elements 4"],
        Int[torch.Tensor, "batch max_elements"],
        Bool[torch.Tensor, "batch max_elements"],
    ]:
        """Pad a layout batch to ``max_num_comp``."""
        if bbox.shape[1] > self.max_num_comp:
            raise ValueError(f"DLT supports at most {self.max_num_comp} elements")

        if mask is None:
            mask = torch.ones(labels.shape, dtype=torch.bool, device=labels.device)
        pad_count = self.max_num_comp - bbox.shape[1]
        if pad_count:
            bbox = torch.cat(
                [
                    bbox,
                    torch.zeros(
                        bbox.shape[0],
                        pad_count,
                        4,
                        dtype=bbox.dtype,
                        device=bbox.device,
                    ),
                ],
                dim=1,
            )
            labels = torch.cat(
                [
                    labels,
                    torch.zeros(
                        labels.shape[0],
                        pad_count,
                        dtype=labels.dtype,
                        device=labels.device,
                    ),
                ],
                dim=1,
            )
            mask = torch.cat(
                [
                    mask,
                    torch.zeros(
                        mask.shape[0], pad_count, dtype=torch.bool, device=mask.device
                    ),
                ],
                dim=1,
            )
        return bbox, labels, mask

    def public_to_internal_boxes(
        self, bbox: Float[torch.Tensor, "batch elements 4"]
    ) -> Float[torch.Tensor, "batch elements 4"]:
        """Map public normalized ``xywh`` boxes to DLT's internal range."""
        return bbox.clamp(0.0, 1.0) * 4.0 - 2.0

    def internal_to_public_boxes(
        self, bbox: Float[torch.Tensor, "batch elements 4"]
    ) -> Float[torch.Tensor, "batch elements 4"]:
        """Map DLT internal-range boxes to public normalized ``xywh``."""
        return (bbox / 2.0 + 1.0).div(2.0).clamp(0.0, 1.0)

    def public_to_internal_labels(
        self,
        labels: Int[torch.Tensor, "batch elements"],
        mask: Bool[torch.Tensor, "batch elements"],
    ) -> Int[torch.Tensor, "batch elements"]:
        """Shift public labels into DLT's internal category ids."""
        shifted = labels.long() + 1
        return torch.where(mask.bool(), shifted, torch.zeros_like(shifted))

    def internal_to_public_labels(
        self,
        labels: Int[torch.Tensor, "batch elements"],
        mask: Bool[torch.Tensor, "batch elements"],
    ) -> Int[torch.Tensor, "batch elements"]:
        """Shift DLT internal category ids back to public dataset-local ids."""
        public = (labels.long() - 1).clamp(0, len(self.labels) - 1)
        return public * mask.long()

    def condition_masks(
        self, condition_type: str, *, mask: Bool[torch.Tensor, "batch elements"]
    ) -> tuple[
        Int[torch.Tensor, "batch elements 4"],
        Int[torch.Tensor, "batch elements"],
    ]:
        """Return DLT ``mask_box`` and ``mask_cat`` tensors.

        ``1`` means generated/noised and ``0`` means conditioned.
        """
        mask_box = torch.ones(
            mask.shape[0], mask.shape[1], 4, dtype=torch.long, device=mask.device
        )
        mask_cat = torch.ones(mask.shape, dtype=torch.long, device=mask.device)
        if condition_type == "label":
            mask_cat.zero_()
        elif condition_type == "label_size":
            mask_box[:, :, 2:] = 0
            mask_cat.zero_()
        elif condition_type == "unconditional":
            pass
        else:
            raise ValueError(f"Unsupported DLT condition_type: {condition_type}")

        mask_box = mask_box * mask.unsqueeze(-1).long()
        mask_cat = mask_cat * mask.long()
        return mask_box, mask_cat

id2label property

id2label: dict[int, str]

Return public label names keyed by dataset-local ids.

categories_num property

categories_num: int

Return internal category count including pad and mask/drop ids.

pad_category_id property

pad_category_id: int

Return the internal padding category id.

mask_category_id property

mask_category_id: int

Return the internal mask/drop category id.

__init__

__init__(
    dataset: DatasetName | str,
    labels: Sequence[str],
    max_num_comp: int,
) -> None

Initialize processor metadata.

Source code in models/dlt/src/dlt/processing_dlt.py
43
44
45
46
47
48
49
50
51
52
53
def __init__(
    self,
    dataset: DatasetName | str,
    labels: Sequence[str],
    max_num_comp: int,
) -> None:
    """Initialize processor metadata."""
    super().__init__()
    self.dataset = str(normalize_dataset(dataset))
    self.labels = tuple(str(label) for label in labels)
    self.max_num_comp = max_num_comp

from_dataset classmethod

from_dataset(dataset: DatasetName | str) -> DLTProcessor

Create a processor from shared dataset metadata.

Source code in models/dlt/src/dlt/processing_dlt.py
55
56
57
58
59
60
61
62
63
64
65
@classmethod
def from_dataset(cls, dataset: DatasetName | str) -> "DLTProcessor":
    """Create a processor from shared dataset metadata."""
    canonical = normalize_dataset(dataset)
    return cls(
        dataset=canonical,
        labels=tuple(default_id2label(canonical).values()),
        max_num_comp=9
        if canonical in {DatasetName.publaynet, DatasetName.rico13}
        else 33,
    )

__call__

__call__(
    *,
    bbox: Float[Tensor, "batch elements 4"]
    | Float[ndarray, "batch elements 4"]
    | LayoutInput,
    labels: Int[Tensor, "batch elements"]
    | Int[ndarray, "batch elements"]
    | LayoutInput,
    mask: Bool[Tensor, "batch elements"]
    | Bool[ndarray, "batch elements"]
    | LayoutInput
    | None = None,
    box_format: BoxFormat | str = BoxFormat.xywh,
    normalized: bool = True,
    canvas_size: tuple[int, int] | None = None,
    device: device | str | None = None,
) -> DLTProcessedBatch

Convert public layout tensors into padded internal tensors.

Source code in models/dlt/src/dlt/processing_dlt.py
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
def __call__(
    self,
    *,
    bbox: Float[torch.Tensor, "batch elements 4"]
    | Float[np.ndarray, "batch elements 4"]
    | LayoutInput,
    labels: Int[torch.Tensor, "batch elements"]
    | Int[np.ndarray, "batch elements"]
    | LayoutInput,
    mask: Bool[torch.Tensor, "batch elements"]
    | Bool[np.ndarray, "batch elements"]
    | LayoutInput
    | None = None,
    box_format: BoxFormat | str = BoxFormat.xywh,
    normalized: bool = True,
    canvas_size: tuple[int, int] | None = None,
    device: torch.device | str | None = None,
) -> DLTProcessedBatch:
    """Convert public layout tensors into padded internal tensors."""
    bbox_t, labels_t, mask_t = prepare_layout_tensors(
        bbox=bbox,
        labels=labels,
        mask=mask,
        box_format=box_format,
        normalized=normalized,
        canvas_size=canvas_size,
    )
    if device is not None:
        bbox_t = bbox_t.to(device)
        labels_t = labels_t.to(device)
        mask_t = mask_t.to(device)
    bbox_t, labels_t, mask_t = self.pad(bbox_t, labels_t, mask_t)
    return cast(
        DLTProcessedBatch,
        {
            "box": self.public_to_internal_boxes(bbox_t) * mask_t.unsqueeze(-1),
            "box_cond": self.public_to_internal_boxes(bbox_t)
            * mask_t.unsqueeze(-1),
            "cat": self.public_to_internal_labels(labels_t, mask_t),
            "mask": mask_t,
        },
    )

empty_condition

empty_condition(
    *,
    batch_size: int,
    device: device | str,
    dtype: dtype = torch.float32,
) -> DLTProcessedBatch

Return an empty unconditional internal batch.

Source code in models/dlt/src/dlt/processing_dlt.py
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
def empty_condition(
    self,
    *,
    batch_size: int,
    device: torch.device | str,
    dtype: torch.dtype = torch.float32,
) -> DLTProcessedBatch:
    """Return an empty unconditional internal batch."""
    device = torch.device(device)
    bbox = torch.zeros(batch_size, self.max_num_comp, 4, dtype=dtype, device=device)
    labels = torch.zeros(
        batch_size, self.max_num_comp, dtype=torch.long, device=device
    )
    mask = torch.ones(
        batch_size, self.max_num_comp, dtype=torch.bool, device=device
    )
    return cast(
        DLTProcessedBatch,
        {
            "box": self.public_to_internal_boxes(bbox),
            "box_cond": self.public_to_internal_boxes(bbox),
            "cat": self.public_to_internal_labels(labels, mask),
            "mask": mask,
        },
    )

pad

pad(
    bbox: Float[Tensor, "batch elements 4"],
    labels: Int[Tensor, "batch elements"],
    mask: Bool[Tensor, "batch elements"] | None = None,
) -> tuple[
    Float[torch.Tensor, "batch max_elements 4"],
    Int[torch.Tensor, "batch max_elements"],
    Bool[torch.Tensor, "batch max_elements"],
]

Pad a layout batch to max_num_comp.

Source code in models/dlt/src/dlt/processing_dlt.py
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
def pad(
    self,
    bbox: Float[torch.Tensor, "batch elements 4"],
    labels: Int[torch.Tensor, "batch elements"],
    mask: Bool[torch.Tensor, "batch elements"] | None = None,
) -> tuple[
    Float[torch.Tensor, "batch max_elements 4"],
    Int[torch.Tensor, "batch max_elements"],
    Bool[torch.Tensor, "batch max_elements"],
]:
    """Pad a layout batch to ``max_num_comp``."""
    if bbox.shape[1] > self.max_num_comp:
        raise ValueError(f"DLT supports at most {self.max_num_comp} elements")

    if mask is None:
        mask = torch.ones(labels.shape, dtype=torch.bool, device=labels.device)
    pad_count = self.max_num_comp - bbox.shape[1]
    if pad_count:
        bbox = torch.cat(
            [
                bbox,
                torch.zeros(
                    bbox.shape[0],
                    pad_count,
                    4,
                    dtype=bbox.dtype,
                    device=bbox.device,
                ),
            ],
            dim=1,
        )
        labels = torch.cat(
            [
                labels,
                torch.zeros(
                    labels.shape[0],
                    pad_count,
                    dtype=labels.dtype,
                    device=labels.device,
                ),
            ],
            dim=1,
        )
        mask = torch.cat(
            [
                mask,
                torch.zeros(
                    mask.shape[0], pad_count, dtype=torch.bool, device=mask.device
                ),
            ],
            dim=1,
        )
    return bbox, labels, mask

public_to_internal_boxes

public_to_internal_boxes(
    bbox: Float[Tensor, "batch elements 4"],
) -> Float[torch.Tensor, "batch elements 4"]

Map public normalized xywh boxes to DLT's internal range.

Source code in models/dlt/src/dlt/processing_dlt.py
210
211
212
213
214
def public_to_internal_boxes(
    self, bbox: Float[torch.Tensor, "batch elements 4"]
) -> Float[torch.Tensor, "batch elements 4"]:
    """Map public normalized ``xywh`` boxes to DLT's internal range."""
    return bbox.clamp(0.0, 1.0) * 4.0 - 2.0

internal_to_public_boxes

internal_to_public_boxes(
    bbox: Float[Tensor, "batch elements 4"],
) -> Float[torch.Tensor, "batch elements 4"]

Map DLT internal-range boxes to public normalized xywh.

Source code in models/dlt/src/dlt/processing_dlt.py
216
217
218
219
220
def internal_to_public_boxes(
    self, bbox: Float[torch.Tensor, "batch elements 4"]
) -> Float[torch.Tensor, "batch elements 4"]:
    """Map DLT internal-range boxes to public normalized ``xywh``."""
    return (bbox / 2.0 + 1.0).div(2.0).clamp(0.0, 1.0)

public_to_internal_labels

public_to_internal_labels(
    labels: Int[Tensor, "batch elements"],
    mask: Bool[Tensor, "batch elements"],
) -> Int[torch.Tensor, "batch elements"]

Shift public labels into DLT's internal category ids.

Source code in models/dlt/src/dlt/processing_dlt.py
222
223
224
225
226
227
228
229
def public_to_internal_labels(
    self,
    labels: Int[torch.Tensor, "batch elements"],
    mask: Bool[torch.Tensor, "batch elements"],
) -> Int[torch.Tensor, "batch elements"]:
    """Shift public labels into DLT's internal category ids."""
    shifted = labels.long() + 1
    return torch.where(mask.bool(), shifted, torch.zeros_like(shifted))

internal_to_public_labels

internal_to_public_labels(
    labels: Int[Tensor, "batch elements"],
    mask: Bool[Tensor, "batch elements"],
) -> Int[torch.Tensor, "batch elements"]

Shift DLT internal category ids back to public dataset-local ids.

Source code in models/dlt/src/dlt/processing_dlt.py
231
232
233
234
235
236
237
238
def internal_to_public_labels(
    self,
    labels: Int[torch.Tensor, "batch elements"],
    mask: Bool[torch.Tensor, "batch elements"],
) -> Int[torch.Tensor, "batch elements"]:
    """Shift DLT internal category ids back to public dataset-local ids."""
    public = (labels.long() - 1).clamp(0, len(self.labels) - 1)
    return public * mask.long()

condition_masks

condition_masks(
    condition_type: str,
    *,
    mask: Bool[Tensor, "batch elements"],
) -> tuple[
    Int[torch.Tensor, "batch elements 4"],
    Int[torch.Tensor, "batch elements"],
]

Return DLT mask_box and mask_cat tensors.

1 means generated/noised and 0 means conditioned.

Source code in models/dlt/src/dlt/processing_dlt.py
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
def condition_masks(
    self, condition_type: str, *, mask: Bool[torch.Tensor, "batch elements"]
) -> tuple[
    Int[torch.Tensor, "batch elements 4"],
    Int[torch.Tensor, "batch elements"],
]:
    """Return DLT ``mask_box`` and ``mask_cat`` tensors.

    ``1`` means generated/noised and ``0`` means conditioned.
    """
    mask_box = torch.ones(
        mask.shape[0], mask.shape[1], 4, dtype=torch.long, device=mask.device
    )
    mask_cat = torch.ones(mask.shape, dtype=torch.long, device=mask.device)
    if condition_type == "label":
        mask_cat.zero_()
    elif condition_type == "label_size":
        mask_box[:, :, 2:] = 0
        mask_cat.zero_()
    elif condition_type == "unconditional":
        pass
    else:
        raise ValueError(f"Unsupported DLT condition_type: {condition_type}")

    mask_box = mask_box * mask.unsqueeze(-1).long()
    mask_cat = mask_cat * mask.long()
    return mask_box, mask_cat

scheduling_dlt

Joint continuous/discrete scheduler for DLT pipelines.

DLTJointSchedulerOutput dataclass

Bases: BaseOutput

Output returned by one joint reverse step.

Source code in models/dlt/src/dlt/scheduling_dlt.py
18
19
20
21
22
23
@dataclass
class DLTJointSchedulerOutput(BaseOutput):
    """Output returned by one joint reverse step."""

    prev_sample: Float[torch.Tensor, "batch elements 4"]
    pred_original_sample: Float[torch.Tensor, "batch elements 4"]

DLTJointDiffusionScheduler

Bases: SchedulerMixin, ConfigMixin

Save/loadable DLT continuous and discrete diffusion scheduler.

Parameters:

Name Type Description Default
alpha float

Probability of changing to a non-mask category.

0.0
beta float

Probability of changing to the mask/drop category.

0.15
seq_max_length int

Maximum number of layout elements.

9
discrete_features_names Sequence[Sequence[str | int]] | None

Discrete feature specs as (name, count).

None
num_discrete_steps Sequence[int] | None

Number of discrete diffusion steps per feature.

None
temperature float

Categorical sampling temperature.

0.8
num_train_timesteps int

Continuous DDPM timesteps.

100
beta_schedule str

Diffusers DDPM beta schedule.

'squaredcos_cap_v2'
prediction_type str

DDPM prediction type.

'sample'
clip_sample bool

Whether DDPM steps clamp predicted samples.

False
Source code in models/dlt/src/dlt/scheduling_dlt.py
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
class DLTJointDiffusionScheduler(SchedulerMixin, ConfigMixin):
    """Save/loadable DLT continuous and discrete diffusion scheduler.

    Args:
        alpha: Probability of changing to a non-mask category.
        beta: Probability of changing to the mask/drop category.
        seq_max_length: Maximum number of layout elements.
        discrete_features_names: Discrete feature specs as ``(name, count)``.
        num_discrete_steps: Number of discrete diffusion steps per feature.
        temperature: Categorical sampling temperature.
        num_train_timesteps: Continuous DDPM timesteps.
        beta_schedule: Diffusers DDPM beta schedule.
        prediction_type: DDPM prediction type.
        clip_sample: Whether DDPM steps clamp predicted samples.
    """

    config_name = "scheduler_config.json"
    order = 1

    @register_to_config
    def __init__(
        self,
        *,
        alpha: float = 0.0,
        beta: float = 0.15,
        seq_max_length: int = 9,
        discrete_features_names: Sequence[Sequence[str | int]] | None = None,
        num_discrete_steps: Sequence[int] | None = None,
        temperature: float = 0.8,
        num_train_timesteps: int = 100,
        beta_schedule: str = "squaredcos_cap_v2",
        prediction_type: str = "sample",
        clip_sample: bool = False,
    ) -> None:
        """Initialize the scheduler and defer transition matrix construction."""
        features = discrete_features_names or _DEFAULT_FEATURES
        steps = list(num_discrete_steps or [10 for _ in features])
        if len(features) != len(steps):
            raise ValueError("Each discrete feature requires a step count")

        self.alpha = alpha
        self.beta = beta
        self.seq_max_length = seq_max_length
        parsed_features: list[DiscreteFeatureSpec] = []
        for raw_feature in features:
            name = str(raw_feature[0])
            raw_count = raw_feature[1]
            if isinstance(raw_count, int):
                count = raw_count
            else:
                count = int(str(raw_count))
            parsed_features.append((name, count))
        self.discrete_features_names = parsed_features
        self.num_discrete_steps = [int(step) for step in steps]
        self.temperature = temperature
        self._cont2disc: dict[str, dict[int, int]] | None = None
        self._transition_matrices: (
            dict[str, list[Float[torch.Tensor, "categories categories"]]] | None
        ) = None
        self._ddpm = DDPMScheduler(
            num_train_timesteps=num_train_timesteps,
            beta_schedule=beta_schedule,
            prediction_type=prediction_type,
            clip_sample=clip_sample,
        )
        self.num_cont_steps = num_train_timesteps
        self.num_train_timesteps = num_train_timesteps
        self.beta_schedule = beta_schedule
        self.prediction_type = prediction_type
        self.clip_sample = clip_sample

    @property
    def cont2disc(self) -> dict[str, dict[int, int]]:
        """Return continuous-to-discrete timestep mappings, computing lazily."""
        if self._cont2disc is None:
            self._cont2disc = {
                name: self.mapping_cont2disc(self.num_train_timesteps, steps)
                for (name, _), steps in zip(
                    self.discrete_features_names, self.num_discrete_steps, strict=True
                )
            }
        return self._cont2disc

    @property
    def transition_matrices(
        self,
    ) -> dict[str, list[Float[torch.Tensor, "categories categories"]]]:
        """Return discrete transition matrices, computing lazily."""
        if self._transition_matrices is None:
            self._transition_matrices = {
                name: self.generate_transition_mat(count, steps)
                for (name, count), steps in zip(
                    self.discrete_features_names, self.num_discrete_steps, strict=True
                )
            }
        return self._transition_matrices

    def add_noise_jointly(
        self,
        vec_cont: Float[torch.Tensor, "batch elements 4"],
        vec_cat: Mapping[str, Float[torch.Tensor, "..."] | Int[torch.Tensor, "..."]],
        timesteps: Int[torch.Tensor, "batch"],
        noise: Float[torch.Tensor, "batch elements 4"],
        generator: torch.Generator | None = None,
    ) -> tuple[
        Float[torch.Tensor, "batch elements 4"],
        dict[str, Int[torch.Tensor, "batch elements"]],
    ]:
        """Add continuous DDPM noise and discrete categorical noise."""
        noised_cont = self._ddpm.add_noise(
            original_samples=vec_cont,
            timesteps=cast(torch.IntTensor, timesteps),
            noise=noise,
        )
        cat_res: dict[str, Int[torch.Tensor, "batch elements"]] = {}
        for f_name, _ in self.discrete_features_names:
            t_to_discrete_stage = [
                self.cont2disc[f_name][int(t.item())] for t in timesteps
            ]
            prob_mat = [
                self.transition_matrices[f_name][u].to(vec_cont.device)[
                    vec_cat[f_name][i]
                ]
                for i, u in enumerate(t_to_discrete_stage)
            ]
            probs = torch.cat(prob_mat)
            cat_noise = torch.multinomial(
                probs, 1, replacement=True, generator=generator
            )
            cat_res[f_name] = rearrange(
                cat_noise, "(d b) 1 -> d b", d=noised_cont.shape[0]
            )
        return noised_cont, cat_res

    def step_jointly(
        self,
        cont_output: Float[torch.Tensor, "batch elements 4"],
        cat_output: dict[str, Float[torch.Tensor, "batch elements categories"]],
        timestep: Int[torch.Tensor, "batch"],
        sample: Float[torch.Tensor, "batch elements 4"],
        generator: torch.Generator | None = None,
        return_dict: bool = True,
    ) -> tuple[DLTJointSchedulerOutput, dict[str, Int[torch.Tensor, "batch elements"]]]:
        """Take one reverse step for boxes and categories."""
        bbox = cast(
            DDPMSchedulerOutput,
            self._ddpm.step(
                cont_output,
                int(timestep.flatten()[0].item()),
                sample,
                generator=generator,
                return_dict=True,
            ),
        )
        bbox_out = DLTJointSchedulerOutput(
            prev_sample=bbox.prev_sample,
            pred_original_sample=cast(torch.Tensor, bbox.pred_original_sample),
        )
        step_cat_res: dict[str, Int[torch.Tensor, "batch elements"]] = {}
        batch_timestep = (
            timestep
            if timestep.numel() == sample.shape[0]
            else timestep.flatten()[0].repeat(sample.shape[0])
        )
        for f_name, f_cat_num in self.discrete_features_names:
            t_to_discrete_stage = [
                self.cont2disc[f_name][int(t.item())] for t in batch_timestep
            ]
            cls, _ = self.denoise_cat(
                cat_output[f_name],
                t_to_discrete_stage,
                f_cat_num,
                self.transition_matrices[f_name],
                generator=generator,
            )
            step_cat_res[f_name] = cls
        return bbox_out, step_cat_res

    def generate_transition_mat(
        self, categories_num: int, num_discrete_steps: int
    ) -> list[Float[torch.Tensor, "categories categories"]]:
        """Generate Markov transition matrices for one discrete feature."""
        transition_mat = (
            np.eye(categories_num) * (1 - self.alpha - self.beta)
            + self.alpha / categories_num
        )
        transition_mat[:, -1] += self.beta
        transition_mat[-1, :] = 0
        transition_mat[-1, -1] = 1
        transition_mat_list: list[Float[torch.Tensor, "categories categories"]] = []
        curr_mat = transition_mat.copy()
        for _ in range(num_discrete_steps):
            transition_mat_list.append(torch.tensor(curr_mat, dtype=torch.float32))
            curr_mat = curr_mat @ transition_mat
        return transition_mat_list

    def denoise_cat(
        self,
        pred: Float[torch.Tensor, "batch elements categories"],
        t: list[int],
        cat_num: int,
        transition_mat_list: list[Float[torch.Tensor, "categories categories"]],
        generator: torch.Generator | None = None,
    ) -> tuple[Int[torch.Tensor, "batch elements"], int]:
        """Denoise a categorical feature using DLT's transition rule."""
        pred_prob = F.softmax(pred, dim=2)
        prob, cls = torch.max(pred_prob, dim=2)
        if t[0] > 1:
            matrix = transition_mat_list[t[0]].to(device=pred.device, dtype=pred.dtype)
            scores = torch.matmul(pred_prob.reshape((-1, cat_num)), matrix)
            scores = scores.reshape(pred_prob.shape)
            scores[:, :, 0] = 0
            logits = scores / self.temperature
            flat = logits.reshape(-1, cat_num)
            res = torch.multinomial(flat, 1, generator=generator).reshape(cls.shape)
        else:
            res = (cat_num - 1) * torch.ones_like(cls, dtype=torch.long)
            top = torch.topk(prob, prob.shape[1], dim=1)
            for row in range(prob.shape[0]):
                res[row, top.indices[row]] = cls[row, top.indices[row]]
        return res, 0

    @staticmethod
    def mapping_cont2disc(
        num_cont_steps: int, num_discrete_steps: int
    ) -> dict[int, int]:
        """Map continuous timesteps onto discrete diffusion stages."""
        block_size = num_cont_steps // num_discrete_steps
        cont2disc: dict[int, int] = {}
        for i in range(num_cont_steps):
            if i >= (num_discrete_steps - 1) * block_size:
                if (
                    num_cont_steps % num_discrete_steps != 0
                    and i >= num_discrete_steps * block_size
                ):
                    cont2disc[i] = num_discrete_steps - 1
                else:
                    cont2disc[i] = i // block_size
            else:
                cont2disc[i] = i // block_size
        return cont2disc

cont2disc property

cont2disc: dict[str, dict[int, int]]

Return continuous-to-discrete timestep mappings, computing lazily.

transition_matrices property

transition_matrices: dict[
    str, list[Float[Tensor, "categories categories"]]
]

Return discrete transition matrices, computing lazily.

__init__

__init__(
    *,
    alpha: float = 0.0,
    beta: float = 0.15,
    seq_max_length: int = 9,
    discrete_features_names: Sequence[Sequence[str | int]]
    | None = None,
    num_discrete_steps: Sequence[int] | None = None,
    temperature: float = 0.8,
    num_train_timesteps: int = 100,
    beta_schedule: str = "squaredcos_cap_v2",
    prediction_type: str = "sample",
    clip_sample: bool = False,
) -> None

Initialize the scheduler and defer transition matrix construction.

Source code in models/dlt/src/dlt/scheduling_dlt.py
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
@register_to_config
def __init__(
    self,
    *,
    alpha: float = 0.0,
    beta: float = 0.15,
    seq_max_length: int = 9,
    discrete_features_names: Sequence[Sequence[str | int]] | None = None,
    num_discrete_steps: Sequence[int] | None = None,
    temperature: float = 0.8,
    num_train_timesteps: int = 100,
    beta_schedule: str = "squaredcos_cap_v2",
    prediction_type: str = "sample",
    clip_sample: bool = False,
) -> None:
    """Initialize the scheduler and defer transition matrix construction."""
    features = discrete_features_names or _DEFAULT_FEATURES
    steps = list(num_discrete_steps or [10 for _ in features])
    if len(features) != len(steps):
        raise ValueError("Each discrete feature requires a step count")

    self.alpha = alpha
    self.beta = beta
    self.seq_max_length = seq_max_length
    parsed_features: list[DiscreteFeatureSpec] = []
    for raw_feature in features:
        name = str(raw_feature[0])
        raw_count = raw_feature[1]
        if isinstance(raw_count, int):
            count = raw_count
        else:
            count = int(str(raw_count))
        parsed_features.append((name, count))
    self.discrete_features_names = parsed_features
    self.num_discrete_steps = [int(step) for step in steps]
    self.temperature = temperature
    self._cont2disc: dict[str, dict[int, int]] | None = None
    self._transition_matrices: (
        dict[str, list[Float[torch.Tensor, "categories categories"]]] | None
    ) = None
    self._ddpm = DDPMScheduler(
        num_train_timesteps=num_train_timesteps,
        beta_schedule=beta_schedule,
        prediction_type=prediction_type,
        clip_sample=clip_sample,
    )
    self.num_cont_steps = num_train_timesteps
    self.num_train_timesteps = num_train_timesteps
    self.beta_schedule = beta_schedule
    self.prediction_type = prediction_type
    self.clip_sample = clip_sample

add_noise_jointly

add_noise_jointly(
    vec_cont: Float[Tensor, "batch elements 4"],
    vec_cat: Mapping[
        str, Float[Tensor, ...] | Int[Tensor, ...]
    ],
    timesteps: Int[Tensor, batch],
    noise: Float[Tensor, "batch elements 4"],
    generator: Generator | None = None,
) -> tuple[
    Float[torch.Tensor, "batch elements 4"],
    dict[str, Int[torch.Tensor, "batch elements"]],
]

Add continuous DDPM noise and discrete categorical noise.

Source code in models/dlt/src/dlt/scheduling_dlt.py
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
def add_noise_jointly(
    self,
    vec_cont: Float[torch.Tensor, "batch elements 4"],
    vec_cat: Mapping[str, Float[torch.Tensor, "..."] | Int[torch.Tensor, "..."]],
    timesteps: Int[torch.Tensor, "batch"],
    noise: Float[torch.Tensor, "batch elements 4"],
    generator: torch.Generator | None = None,
) -> tuple[
    Float[torch.Tensor, "batch elements 4"],
    dict[str, Int[torch.Tensor, "batch elements"]],
]:
    """Add continuous DDPM noise and discrete categorical noise."""
    noised_cont = self._ddpm.add_noise(
        original_samples=vec_cont,
        timesteps=cast(torch.IntTensor, timesteps),
        noise=noise,
    )
    cat_res: dict[str, Int[torch.Tensor, "batch elements"]] = {}
    for f_name, _ in self.discrete_features_names:
        t_to_discrete_stage = [
            self.cont2disc[f_name][int(t.item())] for t in timesteps
        ]
        prob_mat = [
            self.transition_matrices[f_name][u].to(vec_cont.device)[
                vec_cat[f_name][i]
            ]
            for i, u in enumerate(t_to_discrete_stage)
        ]
        probs = torch.cat(prob_mat)
        cat_noise = torch.multinomial(
            probs, 1, replacement=True, generator=generator
        )
        cat_res[f_name] = rearrange(
            cat_noise, "(d b) 1 -> d b", d=noised_cont.shape[0]
        )
    return noised_cont, cat_res

step_jointly

step_jointly(
    cont_output: Float[Tensor, "batch elements 4"],
    cat_output: dict[
        str, Float[Tensor, "batch elements categories"]
    ],
    timestep: Int[Tensor, batch],
    sample: Float[Tensor, "batch elements 4"],
    generator: Generator | None = None,
    return_dict: bool = True,
) -> tuple[
    DLTJointSchedulerOutput,
    dict[str, Int[torch.Tensor, "batch elements"]],
]

Take one reverse step for boxes and categories.

Source code in models/dlt/src/dlt/scheduling_dlt.py
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
def step_jointly(
    self,
    cont_output: Float[torch.Tensor, "batch elements 4"],
    cat_output: dict[str, Float[torch.Tensor, "batch elements categories"]],
    timestep: Int[torch.Tensor, "batch"],
    sample: Float[torch.Tensor, "batch elements 4"],
    generator: torch.Generator | None = None,
    return_dict: bool = True,
) -> tuple[DLTJointSchedulerOutput, dict[str, Int[torch.Tensor, "batch elements"]]]:
    """Take one reverse step for boxes and categories."""
    bbox = cast(
        DDPMSchedulerOutput,
        self._ddpm.step(
            cont_output,
            int(timestep.flatten()[0].item()),
            sample,
            generator=generator,
            return_dict=True,
        ),
    )
    bbox_out = DLTJointSchedulerOutput(
        prev_sample=bbox.prev_sample,
        pred_original_sample=cast(torch.Tensor, bbox.pred_original_sample),
    )
    step_cat_res: dict[str, Int[torch.Tensor, "batch elements"]] = {}
    batch_timestep = (
        timestep
        if timestep.numel() == sample.shape[0]
        else timestep.flatten()[0].repeat(sample.shape[0])
    )
    for f_name, f_cat_num in self.discrete_features_names:
        t_to_discrete_stage = [
            self.cont2disc[f_name][int(t.item())] for t in batch_timestep
        ]
        cls, _ = self.denoise_cat(
            cat_output[f_name],
            t_to_discrete_stage,
            f_cat_num,
            self.transition_matrices[f_name],
            generator=generator,
        )
        step_cat_res[f_name] = cls
    return bbox_out, step_cat_res

generate_transition_mat

generate_transition_mat(
    categories_num: int, num_discrete_steps: int
) -> list[Float[torch.Tensor, "categories categories"]]

Generate Markov transition matrices for one discrete feature.

Source code in models/dlt/src/dlt/scheduling_dlt.py
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
def generate_transition_mat(
    self, categories_num: int, num_discrete_steps: int
) -> list[Float[torch.Tensor, "categories categories"]]:
    """Generate Markov transition matrices for one discrete feature."""
    transition_mat = (
        np.eye(categories_num) * (1 - self.alpha - self.beta)
        + self.alpha / categories_num
    )
    transition_mat[:, -1] += self.beta
    transition_mat[-1, :] = 0
    transition_mat[-1, -1] = 1
    transition_mat_list: list[Float[torch.Tensor, "categories categories"]] = []
    curr_mat = transition_mat.copy()
    for _ in range(num_discrete_steps):
        transition_mat_list.append(torch.tensor(curr_mat, dtype=torch.float32))
        curr_mat = curr_mat @ transition_mat
    return transition_mat_list

denoise_cat

denoise_cat(
    pred: Float[Tensor, "batch elements categories"],
    t: list[int],
    cat_num: int,
    transition_mat_list: list[
        Float[Tensor, "categories categories"]
    ],
    generator: Generator | None = None,
) -> tuple[Int[torch.Tensor, "batch elements"], int]

Denoise a categorical feature using DLT's transition rule.

Source code in models/dlt/src/dlt/scheduling_dlt.py
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
def denoise_cat(
    self,
    pred: Float[torch.Tensor, "batch elements categories"],
    t: list[int],
    cat_num: int,
    transition_mat_list: list[Float[torch.Tensor, "categories categories"]],
    generator: torch.Generator | None = None,
) -> tuple[Int[torch.Tensor, "batch elements"], int]:
    """Denoise a categorical feature using DLT's transition rule."""
    pred_prob = F.softmax(pred, dim=2)
    prob, cls = torch.max(pred_prob, dim=2)
    if t[0] > 1:
        matrix = transition_mat_list[t[0]].to(device=pred.device, dtype=pred.dtype)
        scores = torch.matmul(pred_prob.reshape((-1, cat_num)), matrix)
        scores = scores.reshape(pred_prob.shape)
        scores[:, :, 0] = 0
        logits = scores / self.temperature
        flat = logits.reshape(-1, cat_num)
        res = torch.multinomial(flat, 1, generator=generator).reshape(cls.shape)
    else:
        res = (cat_num - 1) * torch.ones_like(cls, dtype=torch.long)
        top = torch.topk(prob, prob.shape[1], dim=1)
        for row in range(prob.shape[0]):
            res[row, top.indices[row]] = cls[row, top.indices[row]]
    return res, 0

mapping_cont2disc staticmethod

mapping_cont2disc(
    num_cont_steps: int, num_discrete_steps: int
) -> dict[int, int]

Map continuous timesteps onto discrete diffusion stages.

Source code in models/dlt/src/dlt/scheduling_dlt.py
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
@staticmethod
def mapping_cont2disc(
    num_cont_steps: int, num_discrete_steps: int
) -> dict[int, int]:
    """Map continuous timesteps onto discrete diffusion stages."""
    block_size = num_cont_steps // num_discrete_steps
    cont2disc: dict[int, int] = {}
    for i in range(num_cont_steps):
        if i >= (num_discrete_steps - 1) * block_size:
            if (
                num_cont_steps % num_discrete_steps != 0
                and i >= num_discrete_steps * block_size
            ):
                cont2disc[i] = num_discrete_steps - 1
            else:
                cont2disc[i] = i // block_size
        else:
            cont2disc[i] = i // block_size
    return cont2disc

training

Training utilities for DLT.

callbacks

Training callbacks for DLT reference-recipe reproduction.

DLTReferenceEpochSamplingCallback

Bases: Callback

Consume the reference recipe's per-epoch sampling RNG.

Source code in models/dlt/src/dlt/training/callbacks.py
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
class DLTReferenceEpochSamplingCallback(Callback):
    """Consume the reference recipe's per-epoch sampling RNG."""

    def __init__(self, *, num_samples: int = 5) -> None:
        """Store the number of validation layouts sampled after each epoch."""
        self.num_samples = num_samples

    def on_train_epoch_end(self, trainer: Trainer, pl_module: LightningModule) -> None:
        """Run reference-style validation sampling after a training epoch."""
        datamodule = getattr(trainer, "datamodule", None)
        val_data = getattr(datamodule, "val_dataset", None)
        if val_data is None:
            raise RuntimeError(
                "DLTReferenceEpochSamplingCallback requires a datamodule with "
                "a prepared val_dataset"
            )

        consume_reference_epoch_sampling_rng(
            pl_module,
            val_data,
            num_samples=self.num_samples,
        )
__init__
__init__(*, num_samples: int = 5) -> None

Store the number of validation layouts sampled after each epoch.

Source code in models/dlt/src/dlt/training/callbacks.py
62
63
64
def __init__(self, *, num_samples: int = 5) -> None:
    """Store the number of validation layouts sampled after each epoch."""
    self.num_samples = num_samples
on_train_epoch_end
on_train_epoch_end(
    trainer: Trainer, pl_module: LightningModule
) -> None

Run reference-style validation sampling after a training epoch.

Source code in models/dlt/src/dlt/training/callbacks.py
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
def on_train_epoch_end(self, trainer: Trainer, pl_module: LightningModule) -> None:
    """Run reference-style validation sampling after a training epoch."""
    datamodule = getattr(trainer, "datamodule", None)
    val_data = getattr(datamodule, "val_dataset", None)
    if val_data is None:
        raise RuntimeError(
            "DLTReferenceEpochSamplingCallback requires a datamodule with "
            "a prepared val_dataset"
        )

    consume_reference_epoch_sampling_rng(
        pl_module,
        val_data,
        num_samples=self.num_samples,
    )

consume_reference_epoch_sampling_rng

consume_reference_epoch_sampling_rng(
    pl_module: LightningModule,
    val_data: _ReferenceLayoutDataset,
    *,
    num_samples: int = 5,
) -> None

Consume reference post-epoch sampling RNG without logging images.

Source code in models/dlt/src/dlt/training/callbacks.py
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
def consume_reference_epoch_sampling_rng(
    pl_module: LightningModule,
    val_data: _ReferenceLayoutDataset,
    *,
    num_samples: int = 5,
) -> None:
    """Consume reference post-epoch sampling RNG without logging images."""
    if num_samples > len(val_data):
        raise ValueError("num_samples cannot exceed validation dataset length")

    module = cast("DLTTrainingModule", pl_module)
    model = cast(torch.nn.Module, module.model)
    scheduler = cast(_JointScheduler, module.scheduler)
    categories_num = int(module.dlt_config.categories_num)
    device = torch.device(module.device)
    was_training = model.training
    model.eval()
    try:
        indices = np.random.choice(range(len(val_data)), num_samples, replace=False)
        for sample_index, layout_index in enumerate(indices):
            sample = _reference_condition_sample(
                val_data,
                int(layout_index),
                sample_index,
                device=device,
            )
            _sample_from_model(
                sample,
                model,
                scheduler,
                categories_num=categories_num,
                device=device,
            )
    finally:
        if was_training:
            model.train()

config

Small constrained training configuration types for DLT.

DLTSeedMode

Bases: StrEnum

Closed set of DLT training seed modes.

Source code in models/dlt/src/dlt/training/config.py
 8
 9
10
11
12
class DLTSeedMode(StrEnum):
    """Closed set of DLT training seed modes."""

    default = auto()
    deterministic = auto()

datamodule

PyTorch Lightning data module for DLT smoke training.

DLTDataModule

Bases: LightningDataModule

DLT data module with synthetic smoke data by default.

Source code in models/dlt/src/dlt/training/datamodule.py
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
class DLTDataModule(LightningDataModule):
    """DLT data module with synthetic smoke data by default."""

    def __init__(
        self,
        *,
        batch_size: int = 2,
        num_workers: int = 0,
        length: int = 8,
        max_num_comp: int = 4,
        categories_num: int = 7,
        seed: int = 0,
        data_path: str | None = None,
        train_file: str = "publaynet_train.h5",
        val_file: str = "publaynet_val.h5",
        shuffle_train: bool = False,
    ) -> None:
        """Initialize data-module parameters."""
        super().__init__()
        self.batch_size = batch_size
        self.num_workers = num_workers
        self.length = length
        self.max_num_comp = max_num_comp
        self.categories_num = categories_num
        self.seed = seed
        self.data_path = Path(data_path) if data_path is not None else None
        self.train_file = train_file
        self.val_file = val_file
        self.shuffle_train = shuffle_train

    def setup(self, stage: str | None = None) -> None:
        """Create train/validation datasets."""
        del stage
        if self.data_path is not None:
            self.train_dataset = H5DLTDataset(
                self.data_path / self.train_file,
                max_num_comp=self.max_num_comp,
            )
            self.val_dataset = H5DLTDataset(
                self.data_path / self.val_file,
                max_num_comp=self.max_num_comp,
            )
            return
        self.train_dataset = SyntheticDLTDataset(
            length=self.length,
            max_num_comp=self.max_num_comp,
            categories_num=self.categories_num,
            seed=self.seed,
        )
        self.val_dataset = SyntheticDLTDataset(
            length=max(2, self.length // 2),
            max_num_comp=self.max_num_comp,
            categories_num=self.categories_num,
            seed=self.seed + 10_000,
        )

    def train_dataloader(self) -> DataLoader[DLTExample]:
        """Return the train dataloader."""
        if not hasattr(self, "train_dataset"):
            self.setup("fit")
        return DataLoader(
            self.train_dataset,
            batch_size=self.batch_size,
            shuffle=self.shuffle_train,
            num_workers=self.num_workers,
            collate_fn=collate_dlt_batch,
        )

    def val_dataloader(self) -> DataLoader[DLTExample]:
        """Return the validation dataloader."""
        if not hasattr(self, "val_dataset"):
            self.setup("validate")
        return DataLoader(
            self.val_dataset,
            batch_size=self.batch_size,
            shuffle=False,
            num_workers=self.num_workers,
            collate_fn=collate_dlt_batch,
        )
__init__
__init__(
    *,
    batch_size: int = 2,
    num_workers: int = 0,
    length: int = 8,
    max_num_comp: int = 4,
    categories_num: int = 7,
    seed: int = 0,
    data_path: str | None = None,
    train_file: str = "publaynet_train.h5",
    val_file: str = "publaynet_val.h5",
    shuffle_train: bool = False,
) -> None

Initialize data-module parameters.

Source code in models/dlt/src/dlt/training/datamodule.py
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
def __init__(
    self,
    *,
    batch_size: int = 2,
    num_workers: int = 0,
    length: int = 8,
    max_num_comp: int = 4,
    categories_num: int = 7,
    seed: int = 0,
    data_path: str | None = None,
    train_file: str = "publaynet_train.h5",
    val_file: str = "publaynet_val.h5",
    shuffle_train: bool = False,
) -> None:
    """Initialize data-module parameters."""
    super().__init__()
    self.batch_size = batch_size
    self.num_workers = num_workers
    self.length = length
    self.max_num_comp = max_num_comp
    self.categories_num = categories_num
    self.seed = seed
    self.data_path = Path(data_path) if data_path is not None else None
    self.train_file = train_file
    self.val_file = val_file
    self.shuffle_train = shuffle_train
setup
setup(stage: str | None = None) -> None

Create train/validation datasets.

Source code in models/dlt/src/dlt/training/datamodule.py
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
def setup(self, stage: str | None = None) -> None:
    """Create train/validation datasets."""
    del stage
    if self.data_path is not None:
        self.train_dataset = H5DLTDataset(
            self.data_path / self.train_file,
            max_num_comp=self.max_num_comp,
        )
        self.val_dataset = H5DLTDataset(
            self.data_path / self.val_file,
            max_num_comp=self.max_num_comp,
        )
        return
    self.train_dataset = SyntheticDLTDataset(
        length=self.length,
        max_num_comp=self.max_num_comp,
        categories_num=self.categories_num,
        seed=self.seed,
    )
    self.val_dataset = SyntheticDLTDataset(
        length=max(2, self.length // 2),
        max_num_comp=self.max_num_comp,
        categories_num=self.categories_num,
        seed=self.seed + 10_000,
    )
train_dataloader
train_dataloader() -> DataLoader[DLTExample]

Return the train dataloader.

Source code in models/dlt/src/dlt/training/datamodule.py
69
70
71
72
73
74
75
76
77
78
79
def train_dataloader(self) -> DataLoader[DLTExample]:
    """Return the train dataloader."""
    if not hasattr(self, "train_dataset"):
        self.setup("fit")
    return DataLoader(
        self.train_dataset,
        batch_size=self.batch_size,
        shuffle=self.shuffle_train,
        num_workers=self.num_workers,
        collate_fn=collate_dlt_batch,
    )
val_dataloader
val_dataloader() -> DataLoader[DLTExample]

Return the validation dataloader.

Source code in models/dlt/src/dlt/training/datamodule.py
81
82
83
84
85
86
87
88
89
90
91
def val_dataloader(self) -> DataLoader[DLTExample]:
    """Return the validation dataloader."""
    if not hasattr(self, "val_dataset"):
        self.setup("validate")
    return DataLoader(
        self.val_dataset,
        batch_size=self.batch_size,
        shuffle=False,
        num_workers=self.num_workers,
        collate_fn=collate_dlt_batch,
    )

dataset

Small DLT datasets used by smoke training configs and tests.

DLTExample

Bases: TypedDict

One DLT training example with conditioning masks.

Source code in models/dlt/src/dlt/training/dataset.py
14
15
16
17
18
19
20
21
22
class DLTExample(TypedDict):
    """One DLT training example with conditioning masks."""

    box: Float[torch.Tensor, "elements 4"]
    box_cond: Float[torch.Tensor, "elements 4"]
    cat: Int[torch.Tensor, "elements"]
    mask: Bool[torch.Tensor, "elements"]
    mask_box: Int[torch.Tensor, "elements 4"]
    mask_cat: Int[torch.Tensor, "elements"]

DLTStepTrace

Bases: TypedDict

Diagnostic tensors captured from one DLT training step.

Source code in models/dlt/src/dlt/training/dataset.py
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
class DLTStepTrace(TypedDict, total=False):
    """Diagnostic tensors captured from one DLT training step."""

    box: Float[torch.Tensor, "batch elements 4"]
    box_cond: Float[torch.Tensor, "batch elements 4"]
    cat: Int[torch.Tensor, "batch elements"]
    mask: Bool[torch.Tensor, "batch elements"]
    mask_box: Int[torch.Tensor, "batch elements 4"]
    mask_cat: Int[torch.Tensor, "batch elements"]
    noise: Float[torch.Tensor, "batch elements 4"]
    t: Int[torch.Tensor, "batch"]
    noised_box: Float[torch.Tensor, "batch elements 4"]
    noised_cat: Int[torch.Tensor, "batch elements"]
    pred_box: Float[torch.Tensor, "batch elements 4"]
    pred_cat: Float[torch.Tensor, "batch elements categories"]
    masked_l2: Float[torch.Tensor, ""]
    masked_ce: Float[torch.Tensor, ""]
    loss: Float[torch.Tensor, ""]

SyntheticDLTDataset

Bases: Dataset[DLTExample]

Deterministic synthetic DLT batches that never download data.

Source code in models/dlt/src/dlt/training/dataset.py
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
class SyntheticDLTDataset(Dataset[DLTExample]):
    """Deterministic synthetic DLT batches that never download data."""

    def __init__(
        self,
        *,
        length: int = 8,
        max_num_comp: int = 4,
        categories_num: int = 7,
        seed: int = 0,
    ) -> None:
        """Initialize a synthetic dataset."""
        self.length = length
        self.max_num_comp = max_num_comp
        self.categories_num = categories_num
        self.seed = seed

    def __len__(self) -> int:
        """Return dataset length."""
        return self.length

    def __getitem__(self, index: int) -> DLTExample:
        """Return one deterministic synthetic DLT sample."""
        generator = torch.Generator().manual_seed(self.seed + index)
        box = torch.rand(self.max_num_comp, 4, generator=generator) * 4.0 - 2.0
        cat = torch.randint(
            1, self.categories_num - 1, (self.max_num_comp,), generator=generator
        )
        mask = torch.ones(self.max_num_comp, dtype=torch.bool)
        mask_box = torch.ones(self.max_num_comp, 4, dtype=torch.long)
        mask_cat = torch.ones(self.max_num_comp, dtype=torch.long)
        return {
            "box": box.float(),
            "box_cond": box.float(),
            "cat": cat.long(),
            "mask": mask,
            "mask_box": mask_box,
            "mask_cat": mask_cat,
        }
__init__
__init__(
    *,
    length: int = 8,
    max_num_comp: int = 4,
    categories_num: int = 7,
    seed: int = 0,
) -> None

Initialize a synthetic dataset.

Source code in models/dlt/src/dlt/training/dataset.py
48
49
50
51
52
53
54
55
56
57
58
59
60
def __init__(
    self,
    *,
    length: int = 8,
    max_num_comp: int = 4,
    categories_num: int = 7,
    seed: int = 0,
) -> None:
    """Initialize a synthetic dataset."""
    self.length = length
    self.max_num_comp = max_num_comp
    self.categories_num = categories_num
    self.seed = seed
__len__
__len__() -> int

Return dataset length.

Source code in models/dlt/src/dlt/training/dataset.py
62
63
64
def __len__(self) -> int:
    """Return dataset length."""
    return self.length
__getitem__
__getitem__(index: int) -> DLTExample

Return one deterministic synthetic DLT sample.

Source code in models/dlt/src/dlt/training/dataset.py
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
def __getitem__(self, index: int) -> DLTExample:
    """Return one deterministic synthetic DLT sample."""
    generator = torch.Generator().manual_seed(self.seed + index)
    box = torch.rand(self.max_num_comp, 4, generator=generator) * 4.0 - 2.0
    cat = torch.randint(
        1, self.categories_num - 1, (self.max_num_comp,), generator=generator
    )
    mask = torch.ones(self.max_num_comp, dtype=torch.bool)
    mask_box = torch.ones(self.max_num_comp, 4, dtype=torch.long)
    mask_cat = torch.ones(self.max_num_comp, dtype=torch.long)
    return {
        "box": box.float(),
        "box_cond": box.float(),
        "cat": cat.long(),
        "mask": mask,
        "mask_box": mask_box,
        "mask_cat": mask_cat,
    }

H5DLTDataset

Bases: Dataset[DLTExample]

DLT PubLayNet dataset backed by LayoutFlow-style HDF5 files.

Source code in models/dlt/src/dlt/training/dataset.py
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
class H5DLTDataset(Dataset[DLTExample]):
    """DLT PubLayNet dataset backed by LayoutFlow-style HDF5 files."""

    def __init__(
        self,
        path: str | Path,
        *,
        max_num_comp: int = 9,
    ) -> None:
        """Index valid HDF5 rows without loading the full file into memory."""
        self.path = Path(path)
        self.max_num_comp = max_num_comp
        with h5py.File(self.path, "r") as data:
            self.keys = [
                key
                for key in sorted(data.keys(), key=int)
                if 1 < int(data[key]["length"][()]) <= self.max_num_comp
                and 1
                < int(_valid_ltwh_mask(np.asarray(data[key]["bbox"])).sum())
                <= self.max_num_comp
            ]

    def __len__(self) -> int:
        """Return the number of valid layouts."""
        return len(self.keys)

    def __getitem__(self, index: int) -> DLTExample:
        """Return one DLT training sample."""
        box, cat, _, _ = self.get_data_by_ix(index)
        mask_box, mask_cat = _mask_instance(box.shape)
        box, cat, mask_box, mask_cat = _pad_instance(
            box, cat, mask_box, mask_cat, self.max_num_comp
        )
        return {
            "box": torch.tensor(box, dtype=torch.float32),
            "box_cond": torch.tensor(box.copy(), dtype=torch.float32),
            "cat": torch.tensor(cat, dtype=torch.long),
            "mask": torch.tensor(cat != 0, dtype=torch.bool),
            "mask_box": torch.tensor(mask_box, dtype=torch.long),
            "mask_cat": torch.tensor(mask_cat, dtype=torch.long),
        }

    def get_data_by_ix(
        self, index: int
    ) -> tuple[
        Float[np.ndarray, "elements 4"],
        Int[np.ndarray, "elements"],
        list[int],
        str,
    ]:
        """Return one unpadded layout with reference-style element shuffling."""
        with h5py.File(self.path, "r") as data:
            key = self.keys[index]
            row = data[key]
            box = np.asarray(row["bbox"], dtype=np.float32)
            cat = np.asarray(row["categories"], dtype=int)
            length = int(row["length"][()])

        box = box[:length]
        cat = cat[:length]
        valid = _valid_ltwh_mask(box)
        box = box[valid]
        cat = cat[valid]
        length = box.shape[0]

        order = list(range(length))
        random.shuffle(order)
        box = box[order]
        cat = cat[order]
        box = _ltwh_to_scaled_xywh(box)
        return box, cat.astype(np.int64), order, str(key)
__init__
__init__(
    path: str | Path, *, max_num_comp: int = 9
) -> None

Index valid HDF5 rows without loading the full file into memory.

Source code in models/dlt/src/dlt/training/dataset.py
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
def __init__(
    self,
    path: str | Path,
    *,
    max_num_comp: int = 9,
) -> None:
    """Index valid HDF5 rows without loading the full file into memory."""
    self.path = Path(path)
    self.max_num_comp = max_num_comp
    with h5py.File(self.path, "r") as data:
        self.keys = [
            key
            for key in sorted(data.keys(), key=int)
            if 1 < int(data[key]["length"][()]) <= self.max_num_comp
            and 1
            < int(_valid_ltwh_mask(np.asarray(data[key]["bbox"])).sum())
            <= self.max_num_comp
        ]
__len__
__len__() -> int

Return the number of valid layouts.

Source code in models/dlt/src/dlt/training/dataset.py
108
109
110
def __len__(self) -> int:
    """Return the number of valid layouts."""
    return len(self.keys)
__getitem__
__getitem__(index: int) -> DLTExample

Return one DLT training sample.

Source code in models/dlt/src/dlt/training/dataset.py
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
def __getitem__(self, index: int) -> DLTExample:
    """Return one DLT training sample."""
    box, cat, _, _ = self.get_data_by_ix(index)
    mask_box, mask_cat = _mask_instance(box.shape)
    box, cat, mask_box, mask_cat = _pad_instance(
        box, cat, mask_box, mask_cat, self.max_num_comp
    )
    return {
        "box": torch.tensor(box, dtype=torch.float32),
        "box_cond": torch.tensor(box.copy(), dtype=torch.float32),
        "cat": torch.tensor(cat, dtype=torch.long),
        "mask": torch.tensor(cat != 0, dtype=torch.bool),
        "mask_box": torch.tensor(mask_box, dtype=torch.long),
        "mask_cat": torch.tensor(mask_cat, dtype=torch.long),
    }
get_data_by_ix
get_data_by_ix(
    index: int,
) -> tuple[
    Float[np.ndarray, "elements 4"],
    Int[np.ndarray, elements],
    list[int],
    str,
]

Return one unpadded layout with reference-style element shuffling.

Source code in models/dlt/src/dlt/training/dataset.py
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
def get_data_by_ix(
    self, index: int
) -> tuple[
    Float[np.ndarray, "elements 4"],
    Int[np.ndarray, "elements"],
    list[int],
    str,
]:
    """Return one unpadded layout with reference-style element shuffling."""
    with h5py.File(self.path, "r") as data:
        key = self.keys[index]
        row = data[key]
        box = np.asarray(row["bbox"], dtype=np.float32)
        cat = np.asarray(row["categories"], dtype=int)
        length = int(row["length"][()])

    box = box[:length]
    cat = cat[:length]
    valid = _valid_ltwh_mask(box)
    box = box[valid]
    cat = cat[valid]
    length = box.shape[0]

    order = list(range(length))
    random.shuffle(order)
    box = box[order]
    cat = cat[order]
    box = _ltwh_to_scaled_xywh(box)
    return box, cat.astype(np.int64), order, str(key)

collate_dlt_batch

collate_dlt_batch(examples: list[DLTExample]) -> DLTExample

Stack DLT examples into one batch.

Source code in models/dlt/src/dlt/training/dataset.py
289
290
291
292
293
294
295
296
297
298
299
300
def collate_dlt_batch(
    examples: list[DLTExample],
) -> DLTExample:
    """Stack DLT examples into one batch."""
    return {
        "box": torch.stack([example["box"] for example in examples]),
        "box_cond": torch.stack([example["box_cond"] for example in examples]),
        "cat": torch.stack([example["cat"] for example in examples]),
        "mask": torch.stack([example["mask"] for example in examples]),
        "mask_box": torch.stack([example["mask_box"] for example in examples]),
        "mask_cat": torch.stack([example["mask_cat"] for example in examples]),
    }

lightning_module

PyTorch Lightning module for DLT training.

DLTWarmupCosineSchedulerFactory

Create the warmup-cosine scheduler used for DLT training.

Source code in models/dlt/src/dlt/training/lightning_module.py
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
class DLTWarmupCosineSchedulerFactory:
    """Create the warmup-cosine scheduler used for DLT training."""

    def __init__(
        self,
        *,
        num_warmup_steps: int,
        num_training_steps: int | None = None,
        num_cycles: float = 0.5,
        last_epoch: int = -1,
    ) -> None:
        """Store scheduler parameters until the optimizer is available."""
        self.num_warmup_steps = num_warmup_steps
        self.num_training_steps = num_training_steps
        self.num_cycles = num_cycles
        self.last_epoch = last_epoch

    def __call__(
        self,
        optimizer: Optimizer,
        *,
        estimated_stepping_batches: int | None = None,
    ) -> LambdaLR:
        """Build a diffusers warmup-cosine scheduler for an optimizer."""
        num_training_steps = self.num_training_steps
        if num_training_steps is None:
            num_training_steps = estimated_stepping_batches
        if num_training_steps is None:
            raise ValueError(
                "num_training_steps is required unless Lightning estimated "
                "stepping batches are provided"
            )

        return get_cosine_schedule_with_warmup(
            optimizer,
            num_warmup_steps=self.num_warmup_steps,
            num_training_steps=num_training_steps,
            num_cycles=self.num_cycles,
            last_epoch=self.last_epoch,
        )
__init__
__init__(
    *,
    num_warmup_steps: int,
    num_training_steps: int | None = None,
    num_cycles: float = 0.5,
    last_epoch: int = -1,
) -> None

Store scheduler parameters until the optimizer is available.

Source code in models/dlt/src/dlt/training/lightning_module.py
26
27
28
29
30
31
32
33
34
35
36
37
38
def __init__(
    self,
    *,
    num_warmup_steps: int,
    num_training_steps: int | None = None,
    num_cycles: float = 0.5,
    last_epoch: int = -1,
) -> None:
    """Store scheduler parameters until the optimizer is available."""
    self.num_warmup_steps = num_warmup_steps
    self.num_training_steps = num_training_steps
    self.num_cycles = num_cycles
    self.last_epoch = last_epoch
__call__
__call__(
    optimizer: Optimizer,
    *,
    estimated_stepping_batches: int | None = None,
) -> LambdaLR

Build a diffusers warmup-cosine scheduler for an optimizer.

Source code in models/dlt/src/dlt/training/lightning_module.py
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
def __call__(
    self,
    optimizer: Optimizer,
    *,
    estimated_stepping_batches: int | None = None,
) -> LambdaLR:
    """Build a diffusers warmup-cosine scheduler for an optimizer."""
    num_training_steps = self.num_training_steps
    if num_training_steps is None:
        num_training_steps = estimated_stepping_batches
    if num_training_steps is None:
        raise ValueError(
            "num_training_steps is required unless Lightning estimated "
            "stepping batches are provided"
        )

    return get_cosine_schedule_with_warmup(
        optimizer,
        num_warmup_steps=self.num_warmup_steps,
        num_training_steps=num_training_steps,
        num_cycles=self.num_cycles,
        last_epoch=self.last_epoch,
    )

DLTTrainingModule

Bases: LightningModule

Lightning module wrapping DLT's denoising training step.

Source code in models/dlt/src/dlt/training/lightning_module.py
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
class DLTTrainingModule(LightningModule):
    """Lightning module wrapping DLT's denoising training step."""

    def __init__(
        self,
        *,
        config: DLTConfig,
        optimizer: OptimizerCallable = torch.optim.AdamW,
        lr_scheduler: LRSchedulerCallable | None = None,
        loss_box_weight: float = 5.0,
    ) -> None:
        """Initialize the training module."""
        super().__init__()
        self.dlt_config = config
        self.optimizer = optimizer
        self.lr_scheduler = lr_scheduler
        self.loss_box_weight = loss_box_weight
        pipe = build_pipeline(self.dlt_config)
        self.model: DLT = pipe.model
        self.scheduler: DLTJointDiffusionScheduler = pipe.scheduler
        self.latest_step_trace: DLTStepTrace = {}

    def training_step(
        self, batch: DLTExample, batch_idx: int
    ) -> Float[torch.Tensor, ""]:
        """Run one DLT denoising step and return the scalar loss."""
        del batch_idx
        device = self.device
        noise = torch.randn(batch["box"].shape, device=device)
        timesteps = torch.randint(
            0, self.scheduler.num_cont_steps, (batch["box"].shape[0],), device=device
        ).long()
        cont_vec, noisy_batch = self.scheduler.add_noise_jointly(
            batch["box"], {"cat": batch["cat"]}, timesteps, noise
        )
        noisy_batch["box"] = cont_vec
        boxes_predict, cls_predict = self.model(batch, noisy_batch, timesteps)
        loss_mse = masked_l2(batch["box_cond"], boxes_predict, batch["mask_box"])
        loss_cls = masked_cross_entropy(cls_predict, batch["cat"], batch["mask_cat"])
        loss = (self.loss_box_weight * loss_mse + loss_cls).mean()
        self.latest_step_trace = {
            "box": batch["box"].detach(),
            "box_cond": batch["box_cond"].detach(),
            "cat": batch["cat"].detach(),
            "mask_box": batch["mask_box"].detach(),
            "mask_cat": batch["mask_cat"].detach(),
            "noise": noise.detach(),
            "t": timesteps.detach(),
            "noised_box": cont_vec.detach(),
            "noised_cat": noisy_batch["cat"].detach(),
            "pred_box": boxes_predict.detach(),
            "pred_cat": cls_predict.detach(),
            "masked_l2": loss_mse.detach(),
            "masked_ce": loss_cls.detach(),
            "loss": loss.detach(),
        }
        if hasattr(self, "log"):
            self.log("train_loss", loss)
        return loss

    def configure_optimizers(self) -> OptimizerLRScheduler:
        """Create optimizer and optional scheduler from LightningCLI callables."""
        optimizer = self.optimizer(self.parameters())
        if self.lr_scheduler is None:
            return optimizer
        if isinstance(self.lr_scheduler, DLTWarmupCosineSchedulerFactory):
            estimated_stepping_batches = None
            if self.lr_scheduler.num_training_steps is None:
                estimated_stepping_batches = int(
                    self.trainer.estimated_stepping_batches
                )
            scheduler = self.lr_scheduler(
                optimizer,
                estimated_stepping_batches=estimated_stepping_batches,
            )
        else:
            scheduler = self.lr_scheduler(optimizer)
        return {
            "optimizer": optimizer,
            "lr_scheduler": {
                "scheduler": scheduler,
                "interval": "step",
            },
        }
__init__
__init__(
    *,
    config: DLTConfig,
    optimizer: OptimizerCallable = torch.optim.AdamW,
    lr_scheduler: LRSchedulerCallable | None = None,
    loss_box_weight: float = 5.0,
) -> None

Initialize the training module.

Source code in models/dlt/src/dlt/training/lightning_module.py
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
def __init__(
    self,
    *,
    config: DLTConfig,
    optimizer: OptimizerCallable = torch.optim.AdamW,
    lr_scheduler: LRSchedulerCallable | None = None,
    loss_box_weight: float = 5.0,
) -> None:
    """Initialize the training module."""
    super().__init__()
    self.dlt_config = config
    self.optimizer = optimizer
    self.lr_scheduler = lr_scheduler
    self.loss_box_weight = loss_box_weight
    pipe = build_pipeline(self.dlt_config)
    self.model: DLT = pipe.model
    self.scheduler: DLTJointDiffusionScheduler = pipe.scheduler
    self.latest_step_trace: DLTStepTrace = {}
training_step
training_step(
    batch: DLTExample, batch_idx: int
) -> Float[torch.Tensor, ""]

Run one DLT denoising step and return the scalar loss.

Source code in models/dlt/src/dlt/training/lightning_module.py
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
def training_step(
    self, batch: DLTExample, batch_idx: int
) -> Float[torch.Tensor, ""]:
    """Run one DLT denoising step and return the scalar loss."""
    del batch_idx
    device = self.device
    noise = torch.randn(batch["box"].shape, device=device)
    timesteps = torch.randint(
        0, self.scheduler.num_cont_steps, (batch["box"].shape[0],), device=device
    ).long()
    cont_vec, noisy_batch = self.scheduler.add_noise_jointly(
        batch["box"], {"cat": batch["cat"]}, timesteps, noise
    )
    noisy_batch["box"] = cont_vec
    boxes_predict, cls_predict = self.model(batch, noisy_batch, timesteps)
    loss_mse = masked_l2(batch["box_cond"], boxes_predict, batch["mask_box"])
    loss_cls = masked_cross_entropy(cls_predict, batch["cat"], batch["mask_cat"])
    loss = (self.loss_box_weight * loss_mse + loss_cls).mean()
    self.latest_step_trace = {
        "box": batch["box"].detach(),
        "box_cond": batch["box_cond"].detach(),
        "cat": batch["cat"].detach(),
        "mask_box": batch["mask_box"].detach(),
        "mask_cat": batch["mask_cat"].detach(),
        "noise": noise.detach(),
        "t": timesteps.detach(),
        "noised_box": cont_vec.detach(),
        "noised_cat": noisy_batch["cat"].detach(),
        "pred_box": boxes_predict.detach(),
        "pred_cat": cls_predict.detach(),
        "masked_l2": loss_mse.detach(),
        "masked_ce": loss_cls.detach(),
        "loss": loss.detach(),
    }
    if hasattr(self, "log"):
        self.log("train_loss", loss)
    return loss
configure_optimizers
configure_optimizers() -> OptimizerLRScheduler

Create optimizer and optional scheduler from LightningCLI callables.

Source code in models/dlt/src/dlt/training/lightning_module.py
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
def configure_optimizers(self) -> OptimizerLRScheduler:
    """Create optimizer and optional scheduler from LightningCLI callables."""
    optimizer = self.optimizer(self.parameters())
    if self.lr_scheduler is None:
        return optimizer
    if isinstance(self.lr_scheduler, DLTWarmupCosineSchedulerFactory):
        estimated_stepping_batches = None
        if self.lr_scheduler.num_training_steps is None:
            estimated_stepping_batches = int(
                self.trainer.estimated_stepping_batches
            )
        scheduler = self.lr_scheduler(
            optimizer,
            estimated_stepping_batches=estimated_stepping_batches,
        )
    else:
        scheduler = self.lr_scheduler(optimizer)
    return {
        "optimizer": optimizer,
        "lr_scheduler": {
            "scheduler": scheduler,
            "interval": "step",
        },
    }

losses

Masked losses used by DLT training.

masked_l2

masked_l2(
    target: Float[Tensor, "batch elements 4"],
    pred: Float[Tensor, "batch elements 4"],
    mask: Int[Tensor, "batch elements 4"],
) -> Float[torch.Tensor, "batch"]

Return per-example masked squared error.

Source code in models/dlt/src/dlt/training/losses.py
10
11
12
13
14
15
16
17
18
def masked_l2(
    target: Float[torch.Tensor, "batch elements 4"],
    pred: Float[torch.Tensor, "batch elements 4"],
    mask: Int[torch.Tensor, "batch elements 4"],
) -> Float[torch.Tensor, "batch"]:
    """Return per-example masked squared error."""
    loss = F.mse_loss(target, pred, reduction="none")
    denom = mask.sum(dim=(1, 2))
    return (denom > 0) * ((loss * mask.float()).sum(dim=(1, 2)) / (denom + 1e-8))

masked_cross_entropy

masked_cross_entropy(
    pred: Float[Tensor, "batch elements categories"],
    target: Int[Tensor, "batch elements"],
    mask: Int[Tensor, "batch elements"],
) -> Float[torch.Tensor, "batch"]

Return per-example masked category cross entropy.

Source code in models/dlt/src/dlt/training/losses.py
21
22
23
24
25
26
27
28
29
30
31
def masked_cross_entropy(
    pred: Float[torch.Tensor, "batch elements categories"],
    target: Int[torch.Tensor, "batch elements"],
    mask: Int[torch.Tensor, "batch elements"],
) -> Float[torch.Tensor, "batch"]:
    """Return per-example masked category cross entropy."""
    one_hot = F.one_hot(target.long(), num_classes=pred.shape[-1])
    log_probs = F.log_softmax(pred, dim=2)
    loss = (-log_probs * one_hot).sum(dim=2)
    denom = mask.sum(dim=1)
    return (loss * mask.float()).sum(dim=1) / (denom + 0.0001)

parity

DLT S0-S2 parity adapter structures.

DLTStepTrace dataclass

Comparable DLT training-step tensors.

Source code in models/dlt/src/dlt/training/parity.py
12
13
14
15
16
@dataclass(frozen=True)
class DLTStepTrace:
    """Comparable DLT training-step tensors."""

    tensors: DLTStepTraceTensors

DLTSyntheticStepTraceAdapter

Trace adapter for local S0-S2 parity smoke checks.

Source code in models/dlt/src/dlt/training/parity.py
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
class DLTSyntheticStepTraceAdapter:
    """Trace adapter for local S0-S2 parity smoke checks."""

    trace_points = (
        "box",
        "box_cond",
        "cat",
        "mask_box",
        "mask_cat",
        "noise",
        "t",
        "noised_box",
        "noised_cat",
        "pred_box",
        "pred_cat",
        "masked_l2",
        "masked_ce",
        "loss",
    )

    def trace_training_step(
        self, module: DLTTrainingModule, batch: DLTExample
    ) -> DLTStepTrace:
        """Run and collect a DLT training-step trace."""
        module.training_step(batch, 0)
        trace = module.latest_step_trace
        return DLTStepTrace(
            {
                "box": trace["box"],
                "box_cond": trace["box_cond"],
                "cat": trace["cat"],
                "mask_box": trace["mask_box"],
                "mask_cat": trace["mask_cat"],
                "noise": trace["noise"],
                "t": trace["t"],
                "noised_box": trace["noised_box"],
                "noised_cat": trace["noised_cat"],
                "pred_box": trace["pred_box"],
                "pred_cat": trace["pred_cat"],
                "masked_l2": trace["masked_l2"],
                "masked_ce": trace["masked_ce"],
                "loss": trace["loss"],
            }
        )
trace_training_step
trace_training_step(
    module: DLTTrainingModule, batch: DLTExample
) -> DLTStepTrace

Run and collect a DLT training-step trace.

Source code in models/dlt/src/dlt/training/parity.py
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
def trace_training_step(
    self, module: DLTTrainingModule, batch: DLTExample
) -> DLTStepTrace:
    """Run and collect a DLT training-step trace."""
    module.training_step(batch, 0)
    trace = module.latest_step_trace
    return DLTStepTrace(
        {
            "box": trace["box"],
            "box_cond": trace["box_cond"],
            "cat": trace["cat"],
            "mask_box": trace["mask_box"],
            "mask_cat": trace["mask_cat"],
            "noise": trace["noise"],
            "t": trace["t"],
            "noised_box": trace["noised_box"],
            "noised_cat": trace["noised_cat"],
            "pred_box": trace["pred_box"],
            "pred_cat": trace["pred_cat"],
            "masked_l2": trace["masked_l2"],
            "masked_ce": trace["masked_ce"],
            "loss": trace["loss"],
        }
    )

seed

Seed helpers for DLT training.

apply_seed_mode

apply_seed_mode(mode: DLTSeedMode | str, seed: int) -> None

Apply a DLT seed mode to Python, NumPy, and torch.

Source code in models/dlt/src/dlt/training/seed.py
13
14
15
16
17
18
19
20
21
22
23
def apply_seed_mode(mode: DLTSeedMode | str, seed: int) -> None:
    """Apply a DLT seed mode to Python, NumPy, and torch."""
    seed_mode = DLTSeedMode(mode)
    random.seed(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    if torch.cuda.is_available():
        torch.cuda.manual_seed_all(seed)
    if seed_mode is DLTSeedMode.deterministic:
        torch.use_deterministic_algorithms(True, warn_only=True)
        torch.backends.cudnn.benchmark = False