Skip to content

Commit b664e22

Browse files
committed
Simplify sparse checkout file handling
1 parent 27311ed commit b664e22

4 files changed

Lines changed: 27 additions & 44 deletions

File tree

‎mergin/client_pull.py‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525
from .common import CHUNK_SIZE, ClientError, DeltaChangeType, PullActionType
2626
from .models import ProjectDelta, ProjectDeltaChange, PullAction
2727
from .merginproject import MerginProject
28-
from .utils import cleanup_tmp_dir, filter_files, path_matches_filter, save_to_file, validates_file_filter
28+
from .utils import cleanup_tmp_dir, filter_files, is_path_in_scope, save_to_file
2929
from typing import List, Optional
3030

3131
# status = download_project_async(...)
@@ -242,7 +242,6 @@ def _cleanup_failed_download(mergin_project: MerginProject = None):
242242
return dest_path
243243

244244

245-
@validates_file_filter
246245
def download_project_async(mc, project_path, directory, project_version=None, include=None, exclude=None):
247246
"""
248247
Starts project download in background and returns handle to the pending project download.
@@ -253,6 +252,8 @@ def download_project_async(mc, project_path, directory, project_version=None, in
253252
mutually exclusive.
254253
"""
255254

255+
if include and exclude:
256+
raise ClientError("Cannot use both include and exclude filters at the same time")
256257
if "/" not in project_path:
257258
raise ClientError("Project name needs to be fully qualified, e.g. <username>/<projectname>")
258259
if os.path.exists(directory):
@@ -536,7 +537,7 @@ def pull_project_async(mc, directory) -> Optional[PullJob]:
536537
raise
537538

538539
file_filter = mp.file_filter()
539-
delta.changes = [c for c in delta.changes if path_matches_filter(c.path, **file_filter)]
540+
delta.changes = [c for c in delta.changes if is_path_in_scope(c.path, **file_filter)]
540541

541542
mp.log.info(f"got project versions: local version {local_version} / server version {server_version}")
542543

‎mergin/merginproject.py‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
conflicted_copy_file_name,
2626
edit_conflict_file_name,
2727
filter_files,
28+
is_path_in_scope,
2829
)
2930
from .local_changes import FileChange
3031

@@ -212,9 +213,10 @@ def version(self) -> str:
212213
return self._metadata["version"]
213214

214215
def files(self) -> list:
215-
"""Returns project's list of files (each file being a dictionary)"""
216+
"""Returns project's list of files (each file being a dictionary), scoped to this
217+
project's file_filter() if one is set (sparse checkout)."""
216218
self._read_metadata()
217-
return self._metadata["files"]
219+
return filter_files(self._metadata["files"], **self.file_filter())
218220

219221
def file_filter(self) -> dict:
220222
"""
@@ -312,10 +314,12 @@ def ignore_file(self, file):
312314
def inspect_files(self):
313315
"""
314316
Inspect files in project directory and return metadata.
317+
Only files matching this project's file_filter() are included.
315318
316319
:returns: metadata for files in project directory in server required format
317320
:rtype: list[dict]
318321
"""
322+
file_filter = self.file_filter()
319323
files_meta = []
320324
for root, dirs, files in os.walk(self.dir, topdown=True):
321325
dirs[:] = [d for d in dirs if d not in [".mergin"]]
@@ -326,6 +330,8 @@ def inspect_files(self):
326330
abs_path = os.path.abspath(os.path.join(root, file))
327331
rel_path = os.path.relpath(abs_path, start=self.dir)
328332
proj_path = "/".join(rel_path.split(os.path.sep)) # we need posix path
333+
if not is_path_in_scope(proj_path, **file_filter):
334+
continue
329335
files_meta.append(
330336
{
331337
"path": proj_path,
@@ -574,8 +580,7 @@ def get_local_delta(self, diff_directory: str) -> List[ProjectDeltaChange]:
574580
:rtype: List[ProjectDeltaItem]
575581
"""
576582
result = []
577-
current_files = filter_files(self.inspect_files(), **self.file_filter())
578-
changes = self.compare_file_sets(self.files(), current_files)
583+
changes = self.compare_file_sets(self.files(), self.inspect_files())
579584
added = changes.get("added", [])
580585
removed = changes.get("removed", [])
581586
updated = changes.get("updated", [])
@@ -664,8 +669,7 @@ def get_push_changes(self):
664669
:returns: changes metadata for files to be pushed to server
665670
:rtype: dict
666671
"""
667-
current_files = filter_files(self.inspect_files(), **self.file_filter())
668-
changes = self.compare_file_sets(self.files(), current_files)
672+
changes = self.compare_file_sets(self.files(), self.inspect_files())
669673
# do checkpoint to push changes from wal file to gpkg
670674
for file in changes["added"] + changes["updated"]:
671675
size, checksum = do_sqlite_checkpoint(self.fpath(file["path"]), self.log)

‎mergin/test/test_utils.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import pytest
22

3-
from ..utils import path_matches_filter, filter_files
3+
from ..utils import is_path_in_scope, filter_files
44
from ..common import ClientError
55

66

@@ -40,8 +40,8 @@
4040
pytest.param("data.gpkg", None, ["media/*", "*.tmp", "*-wal"], True, id="exclude-list-none-match"),
4141
],
4242
)
43-
def test_path_matches_filter(path, include, exclude, expected):
44-
assert path_matches_filter(path, include=include, exclude=exclude) is expected
43+
def test_is_path_in_scope(path, include, exclude, expected):
44+
assert is_path_in_scope(path, include=include, exclude=exclude) is expected
4545

4646

4747
@pytest.mark.parametrize(
@@ -72,8 +72,8 @@ def test_filter_files_keeps_matching_dicts_as_is():
7272

7373

7474
def test_filter_files_raises_on_mutually_exclusive_args():
75-
"""Unlike path_matches_filter, filter_files() is a public entry point (decorated with
76-
@validates_file_filter) and does enforce that include/exclude are mutually exclusive.
75+
"""Unlike is_path_in_scope, filter_files() is the validating entry point and
76+
enforces that include/exclude are mutually exclusive.
7777
"""
7878
files = [{"path": "a.gpkg"}]
7979
with pytest.raises(ClientError, match="Cannot use both include and exclude"):

‎mergin/utils.py‎

Lines changed: 8 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,6 @@
33
import json
44
import hashlib
55
import fnmatch
6-
import functools
7-
import inspect
86
import re
97
import sqlite3
108
from datetime import datetime
@@ -283,30 +281,14 @@ def is_mergin_config(path: str) -> bool:
283281
return filename == "mergin-config.json"
284282

285283

286-
def validates_file_filter(func):
287-
"""
288-
Marks a function as accepting an `include`/`exclude` glob-filter signature, and validates
289-
those arguments (mutually exclusive) before every call - regardless of whether the caller
290-
passed them positionally or by keyword.
291-
"""
292-
signature = inspect.signature(func)
293-
294-
@functools.wraps(func)
295-
def wrapper(*args, **kwargs):
296-
bound_args = signature.bind_partial(*args, **kwargs)
297-
if bound_args.arguments.get("include") and bound_args.arguments.get("exclude"):
298-
raise ClientError("Cannot use both include and exclude filters at the same time")
299-
return func(*args, **kwargs)
300-
301-
return wrapper
302-
303-
304-
def path_matches_filter(path: str, include: List[str] = None, exclude: List[str] = None) -> bool:
284+
def is_path_in_scope(path: str, include: List[str] = None, exclude: List[str] = None) -> bool:
305285
"""
306286
Returns whether `path` should be kept under a sparse-checkout style include/exclude filter.
307287
308288
With `include`, only paths matching at least one glob pattern are kept. With `exclude`,
309289
paths matching at least one pattern are dropped. With neither given, every path is kept.
290+
291+
Assumes include/exclude were already validated as mutually exclusive by the caller.
310292
"""
311293
if include:
312294
return any(fnmatch.fnmatchcase(path, pattern) for pattern in include)
@@ -315,19 +297,15 @@ def path_matches_filter(path: str, include: List[str] = None, exclude: List[str]
315297
return True
316298

317299

318-
@validates_file_filter
319300
def filter_files(files: List[dict], include: List[str] = None, exclude: List[str] = None) -> List[dict]:
320301
"""
321-
Keep only files matching a sparse-checkout style filter.
322-
323-
:param files: list of file metadata dicts, each with a 'path' key
324-
:param include: glob patterns - only matching files are kept
325-
:param exclude: glob patterns - matching files are dropped
326-
:returns: filtered list of file metadata dicts
302+
Keep only files (dict with 'path' key) matching a sparse-checkout style filter.
327303
328-
.. seealso:: path_matches_filter
304+
.. seealso:: is_path_in_scope
329305
"""
330-
return [f for f in files if path_matches_filter(f["path"], include=include, exclude=exclude)]
306+
if include and exclude:
307+
raise ClientError("Cannot use both include and exclude filters at the same time")
308+
return [f for f in files if is_path_in_scope(f["path"], include=include, exclude=exclude)]
331309

332310

333311
def bytes_to_human_size(bytes: int):

0 commit comments

Comments
 (0)