world-datas-analysis/justfile at master · badele/world-datas-analysis · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
#!/usr/bin/env just -f

set positional-arguments

envname:=`basename $(pwd)`
dockerimage:='badele/world-datas-analysis:latest'

# This help
@help:
    just -l --list-heading=$'{{ file_name(justfile()) }} commands:\n'

###############################################################################
# pre-commit
###############################################################################

# Setup pre-commit
precommit-install:
    #!/usr/bin/env bash
    test ! -f .git/hooks/pre-commit && pre-commit install || true

# Update pre-commit
@precommit-update:
    pre-commit autoupdate

# precommit check
@precommit-check:
    pre-commit run --all-files

# Check requirements
@requirements-check:
    command -v curl >/dev/null 2>&1 || (echo "Please install curl" ; exit 1)
    command -v xz >/dev/null 2>&1 || (echo "Please install xz-utils" ; exit 1)
    command -v sudo >/dev/null 2>&1 || (echo "Please install sudo" ; exit 1)
    docker compose >/dev/null 2>&1 || (echo "Please install docker-compose-v2" ; exit 1)

###############################################################################
# Docker
###############################################################################

# Build the wda docker image
@docker-build:
    docker build -t {{ dockerimage }} .

# Push the wda docker image to docker hub
@docker-push:
    docker push {{ dockerimage }}

# Run duckdb cli on docker
@duckdb:
    duckdb db/wda.duckdb

# Run psql cli on dockers
@psql:
    PGPASSWORD=wda psql -h 127.0.0.1 -U wda -d wda

# Run the wda docker image
@docker-run CMD="": start
    docker run --net host -it --rm -e DATAS_LIST="$DATAS_LIST" -v $(pwd):/wda -v $(pwd)/dataset:/var/lib/postgresql/data/dataset -w /wda {{ dockerimage }} {{ CMD }}

###############################################################################
# DB
###############################################################################

# Reset duckdb database
@db-reset:
    just docker-run "rm -f db/wda.duckdb"

###############################################################################
# dataset
###############################################################################

# Download datasets
@download: requirements-check
    just docker-run ./importer/download.sh

# Update datasets
@update: requirements-check
    just docker-run ./importer/update.sh

# Import datasets to sqlite
@import: requirements-check
    just docker-run ./importer/import.sh

# Lint the project
@lint: requirements-check
    pre-commit run --all-files

# Update documentation
@doc-update FAKEFILENAME:
    DATAS_LIST="" just docker-run 'python3 ./updatedoc.py'

[private]
@perm-grafana:
    mkdir -p grafana-storage
    sudo chown -R 472 grafana
    sudo chown -R 472 dataset
    sudo chown -R 472 grafana-storage

[private]
@perm-user:
    mkdir -p grafana-storage
    sudo chown -R $(id -u) db
    sudo chown -R $(id -u) dataset
    sudo chown -R $(id -u) grafana
    sudo chown -R $(id -u) grafana-storage

# Start grafana
@start: perm-grafana
    docker compose up -d
    echo "go to http://localhost:9300/dashboards"

# Stop grafana
@stop: perm-user
    docker compose stop

# Reset grafana storage
@reset:
    rm -rf grafana-storage

# Open browser to grafana page
@chart: start
    command -v xdg-open > /dev/null && xdg-open http://localhost:9300 || echo "goto to http://localhost:9300/dashboards"

# Open browser to pgadmin page
@pgadmin: start
    command -v xdg-open > /dev/null && xdg-open http://localhost:9080 || echo "goto to http://localhost:9080"

# Inspect parquet file
@parquet-inspect FILE:
    parquet-tools inspect {{ FILE }}

# Convert parquet file to CSV
@parquet-csv FILE:
    parquet-tools csv {{ FILE }}

# Show installed packages
@packages:
    echo $PATH | tr ":" "\n" | grep -E "/nix/store" | sed -e "s/\/nix\/store\/[a-z0-9]\+\-//g" | sed -e "s/\/.*//g"