#  Copyright (c) 2024, Salesforce, Inc.
#  SPDX-License-Identifier: Apache-2
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

import math
from contextlib import contextmanager
from copy import deepcopy
from typing import Any, Generator, Optional

import lightning as L
import numpy as np
import torch
from einops import rearrange, reduce, repeat
from gluonts.model import Input, InputSpec
from gluonts.torch import PyTorchPredictor
from gluonts.transform import (
    AddObservedValuesIndicator,
    AsNumpyArray,
    CausalMeanValueImputation,
    ExpandDimArray,
    TestSplitSampler,
    Transformation,
)
from gluonts.transform.split import TFTInstanceSplitter
from jaxtyping import Bool, Float, Int

from uni2ts.common.torch_util import safe_div
from uni2ts.loss.packed import PackedNLLLoss as _PackedNLLLoss

from .module import MoiraiMoEModule


class SampleNLLLoss(_PackedNLLLoss):
    def reduce_loss(
        self,
        loss: Float[torch.Tensor, "batch seq_len #dim"],
        prediction_mask: Optional[Bool[torch.Tensor, "batch seq_len"]],
        observed_mask: Optional[Bool[torch.Tensor, "batch seq_len #dim"]],
        sample_id: Optional[Int[torch.Tensor, "batch seq_len"]],
        variate_id: Optional[Int[torch.Tensor, "batch seq_len"]],
    ) -> Float[torch.Tensor, "batch"]:
        id_mask = torch.logical_and(
            torch.eq(sample_id.unsqueeze(-1), sample_id.unsqueeze(-2)),
            torch.eq(variate_id.unsqueeze(-1), variate_id.unsqueeze(-2)),
        )
        mask = prediction_mask.unsqueeze(-1) * observed_mask
        tobs = reduce(
            id_mask
            * reduce(
                mask,
                "... seq dim -> ... 1 seq",
                "sum",
            ),
            "... seq1 seq2 -> ... seq1 1",
            "sum",
        )
        loss = safe_div(loss, tobs)
        return (loss * mask).sum(dim=(-1, -2))


class MoiraiMoEForecast(L.LightningModule):
    def __init__(
        self,
        prediction_length: int,
        target_dim: int,
        feat_dynamic_real_dim: int,
        past_feat_dynamic_real_dim: int,
        context_length: int,
        module_kwargs: Optional[dict[str, Any]] = None,
        module: Optional[MoiraiMoEModule] = None,
        patch_size: int = 16,
        num_samples: int = 100,
    ):
        assert (module is not None) or (module_kwargs is not None), (
            "if module is not provided, module_kwargs is required"
        )
        super().__init__()
        self.save_hyperparameters(ignore=["module"])
        self.module = MoiraiMoEModule(**module_kwargs) if module is None else module
        self.per_sample_loss_func = SampleNLLLoss()

    @contextmanager
    def hparams_context(
        self,
        prediction_length: Optional[int] = None,
        target_dim: Optional[int] = None,
        feat_dynamic_real_dim: Optional[int] = None,
        past_feat_dynamic_real_dim: Optional[int] = None,
        context_length: Optional[int] = None,
        patch_size: Optional[int | str] = None,
        num_samples: Optional[int] = None,
    ) -> Generator["MoiraiForecast", None, None]:
        kwargs = {
            "prediction_length": prediction_length,
            "target_dim": target_dim,
            "feat_dynamic_real_dim": feat_dynamic_real_dim,
            "past_feat_dynamic_real_dim": past_feat_dynamic_real_dim,
            "context_length": context_length,
            "patch_size": patch_size,
            "num_samples": num_samples,
        }
        old_hparams = deepcopy(self.hparams)
        for kw, arg in kwargs.items():
            if arg is not None:
                self.hparams[kw] = arg

        yield self

        for kw in kwargs:
            self.hparams[kw] = old_hparams[kw]

    def create_predictor(
        self,
        batch_size: int,
        device: str = "auto",
    ) -> PyTorchPredictor:
        ts_fields = []
        if self.hparams.feat_dynamic_real_dim > 0:
            ts_fields.append("feat_dynamic_real")
            ts_fields.append("observed_feat_dynamic_real")
        past_ts_fields = []
        if self.hparams.past_feat_dynamic_real_dim > 0:
            past_ts_fields.append("past_feat_dynamic_real")
            past_ts_fields.append("past_observed_feat_dynamic_real")

        # This splitter separates the series into past and future
        # Past series consists of the context alone
        instance_splitter = TFTInstanceSplitter(
            instance_sampler=TestSplitSampler(),
            past_length=self.past_length,
            future_length=self.hparams.prediction_length,
            observed_value_field="observed_target",
            time_series_fields=ts_fields,
            past_time_series_fields=past_ts_fields,
        )
        return PyTorchPredictor(
            input_names=self.prediction_input_names,
            prediction_net=self,
            batch_size=batch_size,
            prediction_length=self.hparams.prediction_length,
            input_transform=self.get_default_transform() + instance_splitter,
            device=device,
        )

    def describe_inputs(self, batch_size: int = 1) -> InputSpec:
        data = {
            "past_target": Input(
                shape=(
                    batch_size,
                    self.past_length,
                    self.hparams.target_dim,
                ),
                dtype=torch.float,
            ),
            "past_observed_target": Input(
                shape=(
                    batch_size,
                    self.past_length,
                    self.hparams.target_dim,
                ),
                dtype=torch.bool,
            ),
            "past_is_pad": Input(
                shape=(batch_size, self.past_length),
                dtype=torch.bool,
            ),
        }
        if self.hparams.feat_dynamic_real_dim > 0:
            data["feat_dynamic_real"] = Input(
                shape=(
                    batch_size,
                    self.past_length + self.hparams.prediction_length,
                    self.hparams.feat_dynamic_real_dim,
                ),
                dtype=torch.float,
            )
            data["observed_feat_dynamic_real"] = Input(
                shape=(
                    batch_size,
                    self.past_length + self.hparams.prediction_length,
                    self.hparams.feat_dynamic_real_dim,
                ),
                dtype=torch.bool,
            )
        if self.hparams.past_feat_dynamic_real_dim > 0:
            data["past_feat_dynamic_real"] = Input(
                shape=(
                    batch_size,
                    self.past_length,
                    self.hparams.past_feat_dynamic_real_dim,
                ),
                dtype=torch.float,
            )
            data["past_observed_feat_dynamic_real"] = Input(
                shape=(
                    batch_size,
                    self.past_length,
                    self.hparams.past_feat_dynamic_real_dim,
                ),
                dtype=torch.bool,
            )
        return InputSpec(data=data, zeros_fn=torch.zeros)

    @property
    def prediction_input_names(self) -> list[str]:
        return list(self.describe_inputs())

    @property
    def training_input_names(self):
        return self.prediction_input_names + ["future_target", "future_observed_values"]

    @property
    def past_length(self) -> int:
        return (
            self.hparams.context_length + self.hparams.prediction_length
            if self.hparams.patch_size == "auto"
            else self.hparams.context_length
        )

    def context_token_length(self, patch_size: int) -> int:
        return math.ceil(self.hparams.context_length / patch_size)

    def prediction_token_length(self, patch_size) -> int:
        return math.ceil(self.hparams.prediction_length / patch_size)

    @property
    def max_patch_size(self) -> int:
        return max(self.module.patch_sizes)

    def forward(
        self,
        past_target: Float[torch.Tensor, "batch past_time tgt"],
        past_observed_target: Bool[torch.Tensor, "batch past_time tgt"],
        past_is_pad: Bool[torch.Tensor, "batch past_time"],
        feat_dynamic_real: Optional[Float[torch.Tensor, "batch time feat"]] = None,
        observed_feat_dynamic_real: Optional[
            Float[torch.Tensor, "batch time feat"]
        ] = None,
        past_feat_dynamic_real: Optional[
            Float[torch.Tensor, "batch past_time past_feat"]
        ] = None,
        past_observed_feat_dynamic_real: Optional[
            Float[torch.Tensor, "batch past_time past_feat"]
        ] = None,
        num_samples: Optional[int] = None,
    ) -> Float[torch.Tensor, "batch sample future_time *tgt"]:
        # batch is the batch size
        # past_time is the number of time steps in the past - context length
        # tgt is the target dimension - self.hparams.target_dim
        context_step = self.context_token_length(self.hparams.patch_size)
        context_token = (
            self.hparams.target_dim * context_step
        )  # number of context tokens needed
        predict_step = self.prediction_token_length(self.hparams.patch_size)
        predict_token = (
            self.hparams.target_dim * predict_step
        )  # number of prediction tokens needed

        (
            target,
            observed_mask,
            sample_id,
            time_id,
            variate_id,
            prediction_mask,
        ) = self._convert(
            self.hparams.patch_size,
            past_target,
            past_observed_target,
            past_is_pad,
            feat_dynamic_real=feat_dynamic_real,
            observed_feat_dynamic_real=observed_feat_dynamic_real,
            past_feat_dynamic_real=past_feat_dynamic_real,
            past_observed_feat_dynamic_real=past_observed_feat_dynamic_real,
        )
        patch_size = (
            torch.ones_like(time_id, dtype=torch.long) * self.hparams.patch_size
        )

        pred_index = torch.arange(
            start=context_step - 1, end=context_token, step=context_step
        )
        assign_index = torch.arange(
            start=context_token,
            end=context_token + predict_token,
            step=predict_step,
        )

        if predict_step == 1:
            distr = self.module(
                target,
                observed_mask,
                sample_id,
                time_id,
                variate_id,
                prediction_mask,
                patch_size,
            )
            preds = distr.sample(torch.Size((num_samples or self.hparams.num_samples,)))
            preds[..., assign_index, :] = preds[..., pred_index, :]
            return self._format_preds(
                self.hparams.patch_size, preds, self.hparams.target_dim
            )
        else:
            distr = self.module(
                target,
                observed_mask,
                sample_id,
                time_id,
                variate_id,
                prediction_mask,
                patch_size,
            )
            preds = distr.sample(torch.Size((self.hparams.num_samples,)))

            expand_target = target.unsqueeze(0).repeat(
                self.hparams.num_samples, 1, 1, 1
            )
            expand_prediction_mask = prediction_mask.unsqueeze(0).repeat(
                self.hparams.num_samples, 1, 1
            )
            expand_observed_mask = observed_mask.unsqueeze(0).expand(
                self.hparams.num_samples, -1, -1, -1
            )
            expand_sample_id = sample_id.unsqueeze(0).expand(
                self.hparams.num_samples, -1, -1
            )
            expand_time_id = time_id.unsqueeze(0).expand(
                self.hparams.num_samples, -1, -1
            )
            expand_variate_id = variate_id.unsqueeze(0).expand(
                self.hparams.num_samples, -1, -1
            )
            expand_patch_size = patch_size.unsqueeze(0).expand(
                self.hparams.num_samples, -1, -1
            )

            expand_target[..., assign_index, :] = preds[..., pred_index, :]
            expand_prediction_mask[..., assign_index] = False

            remain_step = predict_step - 1
            while remain_step > 0:
                distr = self.module(
                    expand_target,
                    expand_observed_mask,
                    expand_sample_id,
                    expand_time_id,
                    expand_variate_id,
                    expand_prediction_mask,
                    expand_patch_size,
                )
                preds = distr.sample(torch.Size((1,)))
                _, _, bs, token, ps = preds.shape
                preds = preds.view(-1, bs, token, ps)

                pred_index = assign_index
                assign_index = assign_index + 1
                expand_target[..., assign_index, :] = preds[..., pred_index, :]
                expand_prediction_mask[..., assign_index] = False

                remain_step -= 1

            return self._format_preds(
                self.hparams.patch_size, expand_target, self.hparams.target_dim
            )

    @staticmethod
    def _patched_seq_pad(
        patch_size: int,
        x: torch.Tensor,
        dim: int,
        left: bool = True,
        value: Optional[float] = None,
    ) -> torch.Tensor:
        if dim >= 0:
            dim = -x.ndim + dim
        pad_length = -x.size(dim) % patch_size
        if left:
            pad = (pad_length, 0)
        else:
            pad = (0, pad_length)
        pad = (0, 0) * (abs(dim) - 1) + pad
        return torch.nn.functional.pad(x, pad, value=value)

    def _generate_time_id(
        self,
        patch_size: int,
        past_observed_target: Bool[torch.Tensor, "batch past_seq tgt"],
    ) -> tuple[
        Int[torch.Tensor, "batch past_token"], Int[torch.Tensor, "batch future_token"]
    ]:
        # self._patched_seq_pad pads the past observed target  at the second last dimension
        # so that it is divisible by patch size - so that the second last dimension can be
        # split as (seq patch)
        past_seq_id = reduce(
            self._patched_seq_pad(
                patch_size,
                past_observed_target,
                -2,  # pad on the second last dimension
                left=True,  # pad on the left
            ),
            "... (seq patch) dim -> ... seq",  # pattern of reduction - reduce the patch dimension
            "max",
            patch=patch_size,
        )

        past_seq_id = torch.clamp(
            past_seq_id.cummax(dim=-1).values.cumsum(dim=-1) - 1, min=0
        )
        batch_shape = " ".join(map(str, past_observed_target.shape[:-2]))
        future_seq_id = (
            repeat(
                torch.arange(
                    self.prediction_token_length(patch_size),
                    device=past_observed_target.device,
                ),
                f"prediction -> {batch_shape} prediction",
            )
            + past_seq_id.max(dim=-1, keepdim=True).values
            + 1
        )
        return past_seq_id, future_seq_id

    def _convert(
        self,
        patch_size: int,
        past_target: Float[torch.Tensor, "batch past_time tgt"],
        past_observed_target: Bool[torch.Tensor, "batch past_time tgt"],
        past_is_pad: Bool[torch.Tensor, "batch past_time"],
        future_target: Optional[Float[torch.Tensor, "batch future_time tgt"]] = None,
        future_observed_target: Optional[
            Bool[torch.Tensor, "batch future_time tgt"]
        ] = None,
        future_is_pad: Optional[Bool[torch.Tensor, "batch future_time"]] = None,
        feat_dynamic_real: Optional[Float[torch.Tensor, "batch time feat"]] = None,
        observed_feat_dynamic_real: Optional[
            Float[torch.Tensor, "batch time feat"]
        ] = None,
        past_feat_dynamic_real: Optional[
            Float[torch.Tensor, "batch past_time past_feat"]
        ] = None,
        past_observed_feat_dynamic_real: Optional[
            Float[torch.Tensor, "batch past_time past_feat"]
        ] = None,
    ) -> tuple[
        Float[torch.Tensor, "batch combine_seq patch"],  # target
        Bool[torch.Tensor, "batch combine_seq patch"],  # observed_mask
        Int[torch.Tensor, "batch combine_seq"],  # sample_id
        Int[torch.Tensor, "batch combine_seq"],  # time_id
        Int[torch.Tensor, "batch combine_seq"],  # variate_id
        Bool[torch.Tensor, "batch combine_seq"],  # prediction_mask
    ]:
        batch_shape = past_target.shape[:-2]
        device = past_target.device

        target = []
        observed_mask = []
        sample_id = []
        time_id = []
        variate_id = []
        prediction_mask = []
        dim_count = 0

        past_seq_id, future_seq_id = self._generate_time_id(
            patch_size, past_observed_target
        )

        if future_target is None:  # if future target is not provided, pad with zeros
            future_target = torch.zeros(
                size=batch_shape
                + (
                    self.hparams.prediction_length,
                    past_target.shape[-1],
                ),
                dtype=past_target.dtype,
                device=device,
            )
        # We are using torch.nn.functional.pad to pad the fields so that during training when model tries to check with different patch sizes
        # the weight vector has enough elements to handle the largest of patch_sizes
        target.extend(
            [  # pad the past target on the left
                torch.nn.functional.pad(
                    input=rearrange(
                        self._patched_seq_pad(
                            patch_size, past_target, -2, left=True
                        ),  # pad on the second last dimension
                        "... (seq patch) dim -> ... (dim seq) patch",  # rearrange the tensor
                        patch=patch_size,
                    ),
                    pad=(
                        0,
                        self.max_patch_size - patch_size,
                    ),  # right pad the tensor's last dim with (max_patch - patch)
                ),
                # pad the future target on the right
                torch.nn.functional.pad(
                    input=rearrange(
                        self._patched_seq_pad(
                            patch_size, future_target, -2, left=False
                        ),  # pad on the second last dimension
                        "... (seq patch) dim -> ... (dim seq) patch",  # rearrange the tensor
                        patch=patch_size,
                    ),
                    pad=(
                        0,
                        self.max_patch_size - patch_size,
                    ),  # right pad the tensor's last dim with (max_patch - patch)
                ),
            ]
        )

        if (
            future_observed_target is None
        ):  # if future observed target is not provided, pad with ones - since future target has no missing values
            future_observed_target = torch.ones(
                size=batch_shape
                + (
                    self.hparams.prediction_length,
                    past_observed_target.shape[-1],
                ),
                dtype=torch.bool,
                device=device,
            )
        observed_mask.extend(
            [  # pad the past observed target on the left
                torch.nn.functional.pad(
                    input=rearrange(
                        self._patched_seq_pad(
                            patch_size, past_observed_target, -2, left=True
                        ),  # pad on the second last dimension
                        "... (seq patch) dim -> ... (dim seq) patch",  # rearrange the tensor
                        patch=patch_size,
                    ),
                    pad=(
                        0,
                        self.max_patch_size - patch_size,
                    ),  # right pad the tensor's last dim with (max_patch - patch)
                ),
                # pad the future observed target on the right
                torch.nn.functional.pad(
                    input=rearrange(
                        self._patched_seq_pad(
                            patch_size, future_observed_target, -2, left=False
                        ),  # pad on the second last dimension
                        "... (seq patch) dim -> ... (dim seq) patch",  # rearrange the tensor
                        patch=patch_size,
                    ),
                    pad=(
                        0,
                        self.max_patch_size - patch_size,
                    ),  # right pad the tensor's last dim with (max_patch - patch)
                ),
            ]
        )

        if future_is_pad is None:  # if future is pad is not provided, pad with zeros
            future_is_pad = torch.zeros(
                size=batch_shape + (self.hparams.prediction_length,),
                dtype=torch.long,
                device=device,
            )

        sample_id.extend(
            [  # repeats the reduced tensor values along given dimension to match new dimensions
                repeat(
                    reduce(
                        (
                            self._patched_seq_pad(
                                patch_size, past_is_pad, -1, left=True, value=1
                            )
                            == 0
                        ).int(),
                        "... (seq patch) -> ... seq",
                        "max",
                        patch=patch_size,
                    ),
                    "... seq -> ... (dim seq)",
                    dim=past_target.shape[-1],
                ),
                # repeat the reduced tensor values along given dimension to match new dimensions
                repeat(
                    reduce(
                        (
                            self._patched_seq_pad(
                                patch_size, future_is_pad, -1, left=False, value=1
                            )
                            == 0
                        ).int(),
                        "... (seq patch) -> ... seq",
                        "max",
                        patch=patch_size,
                    ),
                    "... seq -> ... (dim seq)",
                    dim=past_target.shape[-1],
                ),
            ]
        )

        time_id.extend(
            [past_seq_id] * past_target.shape[-1]
            + [future_seq_id] * past_target.shape[-1]
        )

        variate_id.extend(
            [  # torch.arange: 1d tensor of shape (past_target.shape[-1],), range of values (dim_count, dim_count + past_target.shape[-1])
                # repeat: repeat the tensor values along the given dimension to match the new dimensions
                repeat(
                    torch.arange(past_target.shape[-1], device=device) + dim_count,
                    f"dim -> {' '.join(map(str, batch_shape))} (dim past)",
                    past=self.context_token_length(patch_size),
                ),
                # torch.arange: 1d tensor of shape (past_target.shape[-1],), range of values (dim_count, dim_count + past_target.shape[-1])
                # repeat: repeat the tensor values along the given dimension to match the new dimensions
                repeat(
                    torch.arange(past_target.shape[-1], device=device) + dim_count,
                    f"dim -> {' '.join(map(str, batch_shape))} (dim future)",
                    future=self.prediction_token_length(patch_size),
                ),
            ]
        )

        dim_count += past_target.shape[
            -1
        ]  # increment the dim count to track variate_id

        prediction_mask.extend(
            [
                # past is the part we have observed - so we set the prediction mask to zeros
                torch.zeros(
                    size=batch_shape
                    + (self.context_token_length(patch_size) * past_target.shape[-1],),
                    dtype=torch.bool,
                    device=device,
                ),
                # future is thde part we want to predict - so we set the prediction mask to ones
                torch.ones(
                    size=batch_shape
                    + (
                        self.prediction_token_length(patch_size)
                        * past_target.shape[-1],
                    ),
                    dtype=torch.bool,
                    device=device,
                ),
            ]
        )

        if feat_dynamic_real is not None:
            if observed_feat_dynamic_real is None:
                raise ValueError(
                    "observed_feat_dynamic_real must be provided if feat_dynamic_real is provided"
                )

            target.extend(
                [
                    torch.nn.functional.pad(
                        input=rearrange(
                            self._patched_seq_pad(
                                patch_size,
                                feat_dynamic_real[
                                    ..., : self.hparams.context_length, :
                                ],
                                -2,
                                left=True,
                            ),
                            "... (seq patch) dim -> ... (dim seq) patch",
                            patch=patch_size,
                        ),
                        pad=(0, self.max_patch_size - patch_size),
                    ),
                    torch.nn.functional.pad(
                        input=rearrange(
                            self._patched_seq_pad(
                                patch_size,
                                feat_dynamic_real[
                                    ..., self.hparams.context_length :, :
                                ],
                                -2,
                                left=False,
                            ),
                            "... (seq patch) dim -> ... (dim seq) patch",
                            patch=patch_size,
                        ),
                        pad=(0, self.max_patch_size - patch_size),
                    ),
                ]
            )

            observed_mask.extend(
                [
                    torch.nn.functional.pad(
                        input=rearrange(
                            self._patched_seq_pad(
                                patch_size,
                                observed_feat_dynamic_real[
                                    ..., : self.hparams.context_length, :
                                ],
                                -2,
                                left=True,
                            ),
                            "... (seq patch) dim -> ... (dim seq) patch",
                            patch=patch_size,
                        ),
                        pad=(0, self.max_patch_size - patch_size),
                    ),
                    torch.nn.functional.pad(
                        input=rearrange(
                            self._patched_seq_pad(
                                patch_size,
                                observed_feat_dynamic_real[
                                    ..., self.hparams.context_length :, :
                                ],
                                -2,
                                left=False,
                            ),
                            "... (seq patch) dim -> ... (dim seq) patch",
                            patch=patch_size,
                        ),
                        pad=(0, self.max_patch_size - patch_size),
                    ),
                ]
            )

            sample_id.extend(
                [
                    repeat(
                        reduce(
                            (
                                self._patched_seq_pad(
                                    patch_size, past_is_pad, -1, left=True
                                )
                                == 0
                            ).int(),
                            "... (seq patch) -> ... seq",
                            "max",
                            patch=patch_size,
                        ),
                        "... seq -> ... (dim seq)",
                        dim=feat_dynamic_real.shape[-1],
                    ),
                    torch.ones(
                        size=batch_shape
                        + (
                            self.prediction_token_length(patch_size)
                            * feat_dynamic_real.shape[-1],
                        ),
                        dtype=torch.long,
                        device=device,
                    ),
                ]
            )

            time_id.extend(
                [past_seq_id] * feat_dynamic_real.shape[-1]
                + [future_seq_id] * feat_dynamic_real.shape[-1]
            )

            variate_id.extend(
                [
                    repeat(
                        torch.arange(feat_dynamic_real.shape[-1], device=device)
                        + dim_count,
                        f"dim -> {' '.join(map(str, batch_shape))} (dim past)",
                        past=self.context_token_length(patch_size),
                    ),
                    repeat(
                        torch.arange(feat_dynamic_real.shape[-1], device=device)
                        + dim_count,
                        f"dim -> {' '.join(map(str, batch_shape))} (dim future)",
                        future=self.prediction_token_length(patch_size),
                    ),
                ]
            )

            dim_count += feat_dynamic_real.shape[-1]

            prediction_mask.extend(
                [
                    torch.zeros(
                        size=batch_shape
                        + (
                            self.context_token_length(patch_size)
                            * feat_dynamic_real.shape[-1],
                        ),
                        dtype=torch.bool,
                        device=device,
                    ),
                    torch.zeros(
                        size=batch_shape
                        + (
                            self.prediction_token_length(patch_size)
                            * feat_dynamic_real.shape[-1],
                        ),
                        dtype=torch.bool,
                        device=device,
                    ),
                ]
            )

        if past_feat_dynamic_real is not None:
            if past_observed_feat_dynamic_real is None:
                raise ValueError(
                    "past_observed_feat_dynamic_real must be provided if past_feat_dynamic_real is provided"
                )

            target.append(
                torch.nn.functional.pad(
                    input=rearrange(
                        self._patched_seq_pad(
                            patch_size, past_feat_dynamic_real, -2, left=True
                        ),
                        "... (seq patch) dim -> ... (dim seq) patch",
                        patch=patch_size,
                    ),
                    pad=(0, self.max_patch_size - patch_size),
                )
            )

            observed_mask.append(
                torch.nn.functional.pad(
                    input=rearrange(
                        self._patched_seq_pad(
                            patch_size, past_observed_feat_dynamic_real, -2, left=True
                        ),
                        "... (seq patch) dim -> ... (dim seq) patch",
                        patch=patch_size,
                    ),
                    pad=(0, self.max_patch_size - patch_size),
                )
            )

            sample_id.append(
                repeat(
                    reduce(
                        (
                            self._patched_seq_pad(
                                patch_size, past_is_pad, -1, left=True
                            )
                            == 0
                        ).int(),
                        "... (seq patch) -> ... seq",
                        "max",
                        patch=patch_size,
                    ),
                    "... seq -> ... (dim seq)",
                    dim=past_feat_dynamic_real.shape[-1],
                )
            )

            time_id.extend([past_seq_id] * past_feat_dynamic_real.shape[-1])

            variate_id.append(
                repeat(
                    torch.arange(past_feat_dynamic_real.shape[-1], device=device)
                    + dim_count,
                    f"dim -> {' '.join(map(str, batch_shape))} (dim past)",
                    past=self.context_token_length(patch_size),
                )
            )

            dim_count += past_feat_dynamic_real.shape[-1]

            prediction_mask.append(
                torch.zeros(
                    size=batch_shape
                    + (
                        self.context_token_length(patch_size)
                        * past_feat_dynamic_real.shape[-1],
                    ),
                    dtype=torch.bool,
                    device=device,
                )
            )

        target = torch.cat(target, dim=-2)  # (16 8 16) + (16 4 16) --> (16 12 16)
        observed_mask = torch.cat(
            observed_mask, dim=-2
        )  # (16 8 16) + (16 4 16) --> (16 12 16)
        sample_id = torch.cat(sample_id, dim=-1)  # (16 8) + (16 4) --> (16 12)
        time_id = torch.cat(time_id, dim=-1)  # (16 8) + (16 4) --> (16 12)
        variate_id = torch.cat(variate_id, dim=-1)  # (16 8) + (16 4) --> (16 12)
        prediction_mask = torch.cat(
            prediction_mask, dim=-1
        )  # (16 8) + (16 4) --> (16 12)

        return (
            target,
            observed_mask,
            sample_id,
            time_id,
            variate_id,
            prediction_mask,
        )

    def _format_preds(
        self,
        patch_size: int,
        preds: Float[torch.Tensor, "sample batch combine_seq patch"],
        target_dim: int,
    ) -> Float[torch.Tensor, "batch sample future_time *tgt"]:
        start = target_dim * self.context_token_length(patch_size)
        end = start + target_dim * self.prediction_token_length(patch_size)
        preds = preds[..., start:end, :patch_size]
        preds = rearrange(
            preds,
            "sample ... (dim seq) patch -> ... sample (seq patch) dim",
            dim=target_dim,
        )[..., : self.hparams.prediction_length, :]
        return preds.squeeze(-1)

    def get_default_transform(self) -> Transformation:
        # AsNumpyArray casts the input to a numpy array with the specified dtype
        # AddObservedValuesIndicator adds a boolean mask for observed values (True for observed, False for missing)
        # CausalMeanValueImputation imputes missing values with the mean of the observed values
        # ExpandDimArray adds a new dimension to the input array
        transform = AsNumpyArray(
            field="target",
            expected_ndim=1 if self.hparams.target_dim == 1 else 2,
            dtype=np.float32,
        )
        if self.hparams.target_dim == 1:
            transform += AddObservedValuesIndicator(
                target_field="target",
                output_field="observed_target",
                imputation_method=CausalMeanValueImputation(),
                dtype=bool,
            )
            transform += ExpandDimArray(field="target", axis=0)
            transform += ExpandDimArray(field="observed_target", axis=0)
        else:
            transform += AddObservedValuesIndicator(
                target_field="target",
                output_field="observed_target",
                dtype=bool,
            )

        if self.hparams.feat_dynamic_real_dim > 0:
            transform += AsNumpyArray(
                field="feat_dynamic_real",
                expected_ndim=2,
                dtype=np.float32,
            )
            transform += AddObservedValuesIndicator(
                target_field="feat_dynamic_real",
                output_field="observed_feat_dynamic_real",
                dtype=bool,
            )

        if self.hparams.past_feat_dynamic_real_dim > 0:
            transform += AsNumpyArray(
                field="past_feat_dynamic_real",
                expected_ndim=2,
                dtype=np.float32,
            )
            transform += AddObservedValuesIndicator(
                target_field="past_feat_dynamic_real",
                output_field="past_observed_feat_dynamic_real",
                dtype=bool,
            )
        return transform
