Skip to content

dataset

create_dataset(project_id, dataset, user=Depends(current_user_act_ver_prof), db=Depends(get_async_db)) async

Add new dataset to current project

Source code in fractal_server/app/routes/api/v2/dataset.py 
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
@router.post(
    "/project/{project_id}/dataset/",
    response_model=DatasetRead,
    status_code=status.HTTP_201_CREATED,
)
async def create_dataset(
    project_id: int,
    dataset: DatasetCreate,
    user: UserOAuth = Depends(current_user_act_ver_prof),
    db: AsyncSession = Depends(get_async_db),
) -> DatasetRead | None:
    """
    Add new dataset to current project
    """
    project = await _get_project_check_access(
        project_id=project_id,
        user_id=user.id,
        required_permissions=ProjectPermissions.WRITE,
        db=db,
    )

    db_dataset = DatasetV2(
        project_id=project_id,
        zarr_dir="__PLACEHOLDER__",
        **dataset.model_dump(exclude={"project_dir", "zarr_subfolder"}),
    )
    db.add(db_dataset)
    await db.commit()
    await db.refresh(db_dataset)

    if dataset.project_dir is None:
        project_dir = user.project_dirs[0]
    else:
        if dataset.project_dir not in user.project_dirs:
            await db.delete(db_dataset)
            await db.commit()
            raise HTTPException(
                status_code=status.HTTP_403_FORBIDDEN,
                detail=f"You are not allowed to use {dataset.project_dir=}.",
            )
        project_dir = dataset.project_dir

    if dataset.zarr_subfolder is None:
        zarr_subfolder = (
            f"fractal/{project_id}_{sanitize_string(project.name)}/"
            f"{db_dataset.id}_{sanitize_string(db_dataset.name)}"
        )
    else:
        zarr_subfolder = dataset.zarr_subfolder

    zarr_dir = os.path.join(project_dir, zarr_subfolder)
    db_dataset.zarr_dir = normalize_url(zarr_dir)

    db.add(db_dataset)
    await db.commit()
    await db.refresh(db_dataset)

    return db_dataset

delete_dataset(project_id, dataset_id, user=Depends(current_user_act_ver_prof), db=Depends(get_async_db)) async

Delete a dataset associated to the current project

Source code in fractal_server/app/routes/api/v2/dataset.py 
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
@router.delete(
    "/project/{project_id}/dataset/{dataset_id}/",
    status_code=204,
)
async def delete_dataset(
    project_id: int,
    dataset_id: int,
    user: UserOAuth = Depends(current_user_act_ver_prof),
    db: AsyncSession = Depends(get_async_db),
) -> Response:
    """
    Delete a dataset associated to the current project
    """
    output = await _get_dataset_check_access(
        project_id=project_id,
        dataset_id=dataset_id,
        user_id=user.id,
        required_permissions=ProjectPermissions.WRITE,
        db=db,
    )
    dataset = output["dataset"]

    # Fail if there exist jobs that are submitted and in relation with the
    # current dataset.
    stm = _get_submitted_jobs_statement().where(JobV2.dataset_id == dataset_id)
    res = await db.execute(stm)
    jobs = res.scalars().all()
    if jobs:
        string_ids = str([job.id for job in jobs])[1:-1]
        raise HTTPException(
            status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
            detail=(
                f"Cannot delete dataset {dataset.id} because it "
                f"is linked to active job(s) {string_ids}."
            ),
        )

    # Delete dataset
    await db.delete(dataset)
    await db.commit()

    return Response(status_code=status.HTTP_204_NO_CONTENT)

export_dataset(project_id, dataset_id, user=Depends(current_user_act_ver_prof), db=Depends(get_async_db)) async

Export an existing dataset

Source code in fractal_server/app/routes/api/v2/dataset.py 
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
@router.get(
    "/project/{project_id}/dataset/{dataset_id}/export/",
    response_model=DatasetExport,
)
async def export_dataset(
    project_id: int,
    dataset_id: int,
    user: UserOAuth = Depends(current_user_act_ver_prof),
    db: AsyncSession = Depends(get_async_db),
) -> DatasetExport | None:
    """
    Export an existing dataset
    """
    dict_dataset_project = await _get_dataset_check_access(
        project_id=project_id,
        dataset_id=dataset_id,
        user_id=user.id,
        required_permissions=ProjectPermissions.READ,
        db=db,
    )
    dataset = dict_dataset_project["dataset"]

    return dataset

import_dataset(project_id, dataset, user=Depends(current_user_act_ver_prof), db=Depends(get_async_db)) async

Import an existing dataset into a project

Source code in fractal_server/app/routes/api/v2/dataset.py 
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
@router.post(
    "/project/{project_id}/dataset/import/",
    response_model=DatasetRead,
    status_code=status.HTTP_201_CREATED,
)
async def import_dataset(
    project_id: int,
    dataset: DatasetImport,
    user: UserOAuth = Depends(current_user_act_ver_prof),
    db: AsyncSession = Depends(get_async_db),
) -> DatasetRead | None:
    """
    Import an existing dataset into a project
    """

    # Preliminary checks
    await _get_project_check_access(
        project_id=project_id,
        user_id=user.id,
        required_permissions=ProjectPermissions.WRITE,
        db=db,
    )

    if not any(
        Path(dataset.zarr_dir).is_relative_to(project_dir)
        for project_dir in user.project_dirs
    ):
        raise HTTPException(
            status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
            detail=(
                f"{dataset.zarr_dir=} is not relative to any of user's project "
                "dirs."
            ),
        )

    # Create new Dataset
    db_dataset = DatasetV2(
        project_id=project_id,
        **dataset.model_dump(exclude_none=True),
    )
    db.add(db_dataset)
    await db.commit()
    await db.refresh(db_dataset)

    return db_dataset

read_dataset(project_id, dataset_id, user=Depends(current_user_act_ver_prof), db=Depends(get_async_db)) async

Get info on a dataset associated to the current project

Source code in fractal_server/app/routes/api/v2/dataset.py 
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
@router.get(
    "/project/{project_id}/dataset/{dataset_id}/",
    response_model=DatasetRead,
)
async def read_dataset(
    project_id: int,
    dataset_id: int,
    user: UserOAuth = Depends(current_user_act_ver_prof),
    db: AsyncSession = Depends(get_async_db),
) -> DatasetRead | None:
    """
    Get info on a dataset associated to the current project
    """
    output = await _get_dataset_check_access(
        project_id=project_id,
        dataset_id=dataset_id,
        user_id=user.id,
        required_permissions=ProjectPermissions.READ,
        db=db,
    )
    dataset = output["dataset"]
    return dataset

read_dataset_list(project_id, user=Depends(current_user_act_ver_prof), db=Depends(get_async_db)) async

Get dataset list for given project

Source code in fractal_server/app/routes/api/v2/dataset.py 
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
@router.get(
    "/project/{project_id}/dataset/",
    response_model=list[DatasetRead],
)
async def read_dataset_list(
    project_id: int,
    user: UserOAuth = Depends(current_user_act_ver_prof),
    db: AsyncSession = Depends(get_async_db),
) -> list[DatasetRead] | None:
    """
    Get dataset list for given project
    """
    # Access control
    project = await _get_project_check_access(
        project_id=project_id,
        user_id=user.id,
        required_permissions=ProjectPermissions.READ,
        db=db,
    )
    # Find datasets of the current project. Note: this select/where approach
    # has much better scaling than refreshing all elements of
    # `project.dataset_list` - ref
    # https://github.com/fractal-analytics-platform/fractal-server/pull/1082#issuecomment-1856676097.
    stm = select(DatasetV2).where(DatasetV2.project_id == project.id)
    res = await db.execute(stm)
    dataset_list = res.scalars().all()
    return dataset_list

update_dataset(project_id, dataset_id, dataset_update, user=Depends(current_user_act_ver_prof), db=Depends(get_async_db)) async

Edit a dataset associated to the current project

Source code in fractal_server/app/routes/api/v2/dataset.py 
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
@router.patch(
    "/project/{project_id}/dataset/{dataset_id}/",
    response_model=DatasetRead,
)
async def update_dataset(
    project_id: int,
    dataset_id: int,
    dataset_update: DatasetUpdate,
    user: UserOAuth = Depends(current_user_act_ver_prof),
    db: AsyncSession = Depends(get_async_db),
) -> DatasetRead | None:
    """
    Edit a dataset associated to the current project
    """

    output = await _get_dataset_check_access(
        project_id=project_id,
        dataset_id=dataset_id,
        user_id=user.id,
        required_permissions=ProjectPermissions.WRITE,
        db=db,
    )
    db_dataset = output["dataset"]

    for key, value in dataset_update.model_dump(exclude_unset=True).items():
        setattr(db_dataset, key, value)

    await db.commit()
    await db.refresh(db_dataset)
    return db_dataset