Skip to content

Commit e25c856

Browse files
authored
feat: crag exporter job (#379)
* feat: crag exporter
1 parent d649c6d commit e25c856

11 files changed

Lines changed: 288 additions & 47 deletions

File tree

.vscode/launch.json

Lines changed: 24 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,19 @@
44
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
55
"version": "0.2.0",
66
"configurations": [
7+
{
8+
"type": "node",
9+
"request": "launch",
10+
"name": "Crag Geojson",
11+
"program": "${workspaceFolder}/src/db/export/CragGeojson/index.ts",
12+
"preLaunchTask": "tsc: build - tsconfig.json",
13+
"outFiles": [
14+
"${workspaceFolder}/build/**/*.js"
15+
],
16+
"skipFiles": [
17+
"<node_internals>/**"
18+
],
19+
},
720
{
821
"type": "node",
922
"request": "launch",
@@ -48,14 +61,21 @@
4861
"type": "node",
4962
"request": "launch",
5063
"name": "Launch API Server (serve-dev)",
51-
"skipFiles": ["<node_internals>/**"],
64+
"skipFiles": [
65+
"<node_internals>/**"
66+
],
5267
"program": "${workspaceFolder}/src/main.ts",
5368
"preLaunchTask": "tsc: build - tsconfig.json",
54-
"outFiles": ["${workspaceFolder}/build/**/*.js"],
69+
"outFiles": [
70+
"${workspaceFolder}/build/**/*.js"
71+
],
5572
"runtimeExecutable": "yarn",
56-
"runtimeArgs": ["run", "serve-dev"],
73+
"runtimeArgs": [
74+
"run",
75+
"serve-dev"
76+
],
5777
"console": "integratedTerminal"
58-
},
78+
},
5979
{
6080
"name": "Debug Jest Tests",
6181
"type": "node",

export-crag-data.sh

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
#!/bin/bash
2+
3+
if [ -z ${GITHUB_ACCESS_TOKEN} ]
4+
then
5+
echo "GITHUB_ACCESS_TOKEN not defined."
6+
exit 1
7+
fi
8+
9+
echo "cloning openbeta-export repository"
10+
git clone --depth 1 --branch production https://ob-bot-user:${GITHUB_ACCESS_TOKEN}@github.com/OpenBeta/openbeta-export || exit 1
11+
git config user.name "db-export-bot"
12+
git config user.email "db-export-bot@noreply"
13+
cd ..
14+
15+
echo "start exporting CRAG data..."
16+
yarn export-crags
17+
18+
echo "... finished export. Committing data..."
19+
20+
git add -A
21+
git commit -am "export crag data"
22+
git push origin production

package.json

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,9 @@
3131
"@turf/area": "^6.5.0",
3232
"@turf/bbox": "^6.5.0",
3333
"@turf/bbox-polygon": "^6.5.0",
34-
"@turf/centroid": "^6.5.0",
3534
"@turf/circle": "^6.5.0",
35+
"@turf/convex": "^6.5.0",
36+
"@turf/helpers": "^6.5.0",
3637
"@types/uuid": "^8.3.3",
3738
"apollo-datasource-mongodb": "^0.5.4",
3839
"apollo-server": "^3.9.0",
@@ -83,7 +84,8 @@
8384
"export:json:full": "yarn build && node build/db/export/json/index.js",
8485
"export-prod": "./export.sh",
8586
"prepare": "husky install",
86-
"import-users": "tsc ; node build/db/utils/jobs/migration/CreateUsersCollection.js"
87+
"import-users": "tsc ; node build/db/utils/jobs/migration/CreateUsersCollection.js",
88+
"export-crags": "tsc ; node build/db/utils/jobs/CragGeojson/index.js"
8789
},
8890
"standard": {
8991
"plugins": [
@@ -103,4 +105,4 @@
103105
"engines": {
104106
"node": ">=16.14.0"
105107
}
106-
}
108+
}

src/db/AreaSchema.ts

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,20 @@ import { GradeContexts } from '../GradeUtils.js'
88

99
const { Schema, connection } = mongoose
1010

11+
const polygonSchema = new mongoose.Schema({
12+
type: {
13+
type: String,
14+
enum: ['Polygon'],
15+
required: true
16+
},
17+
coordinates: {
18+
type: [[[Number]]], // Array of arrays of arrays of numbers
19+
required: true
20+
}
21+
}, {
22+
_id: false
23+
})
24+
1125
const ChangeRecordMetadata = new Schema<ChangeRecordMetadataType>({
1226
user: {
1327
type: 'object',
@@ -32,6 +46,7 @@ const MetadataSchema = new Schema<IAreaMetadata>({
3246
type: PointSchema,
3347
index: '2dsphere'
3448
},
49+
polygon: polygonSchema,
3550
bbox: [{ type: Number, required: true }],
3651
leftRightIndex: { type: Number, required: false },
3752
ext_id: { type: String, required: false, index: true },
@@ -121,13 +136,18 @@ AreaSchema.index({ _deleting: 1 }, { expireAfterSeconds: 0 })
121136
AreaSchema.index({
122137
'metadata.leftRightIndex': 1
123138
}, {
139+
name: 'leftRightIndex',
124140
partialFilterExpression: {
125141
'metadata.leftRightIndex': {
126142
$gt: -1
127143
}
128144
}
129145
})
130146

147+
AreaSchema.index({
148+
children: 1
149+
})
150+
131151
export const createAreaModel = (name: string = 'areas'): mongoose.Model<AreaType> => {
132152
return connection.model(name, AreaSchema)
133153
}

src/db/AreaTypes.ts

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import mongoose from 'mongoose'
22
import { MUUID } from 'uuid-mongodb'
33

4-
import { BBox, Point } from '@turf/helpers'
4+
import { BBox, Point, Polygon } from '@turf/helpers'
55
import { ClimbType } from './ClimbTypes.js'
66
import { ChangeRecordMetadataType } from './ChangeLogType.js'
77
import { GradeContexts } from '../GradeUtils.js'
@@ -118,8 +118,7 @@ export interface IAreaMetadata {
118118
*/
119119
isBoulder?: boolean
120120
/**
121-
* Areas may be very large, and this point may represent the centroid of the area's bounds
122-
* or a spec point chosen by users.
121+
* Location of a wall or a boulder aka leaf node. Use `bbox` or `polygon` non-leaf areas.
123122
* */
124123
lnglat: Point
125124
/**
@@ -143,6 +142,11 @@ export interface IAreaMetadata {
143142
* GQL layer use these values for querying and identification of areas.
144143
*/
145144
area_id: MUUID
145+
146+
/**
147+
* A polygon (created by convex hull) containing all child areas.
148+
*/
149+
polygon?: Polygon
146150
}
147151
export interface IAreaContent {
148152
/** longform to mediumform description of this area.

src/db/index.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ export const connectDB = async (onConnected: () => any = defaultFn): Promise<voi
5555

5656
await mongoose.connect(
5757
`${scheme}://${user}:${pass}@${server}/${dbName}?authSource=${authDb}&tls=${tlsFlag}&replicaSet=${rsName}`,
58-
{ autoIndex: false }
58+
{ autoIndex: true }
5959
)
6060
} catch (e) {
6161
logger.error("Can't connect to db")
Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
import { createWriteStream } from 'node:fs'
2+
import { point, feature, featureCollection, Feature, Point, Polygon } from '@turf/helpers'
3+
import os from 'node:os'
4+
import { MUUID } from 'uuid-mongodb'
5+
6+
import { connectDB, gracefulExit, getAreaModel } from '../../../index.js'
7+
import { logger } from '../../../../logger.js'
8+
9+
/**
10+
* Export leaf areas as Geojson. Leaf areas are crags/boulders that have climbs.
11+
*/
12+
async function exportLeafCrags (): Promise<void> {
13+
const model = getAreaModel()
14+
15+
const stream = createWriteStream('crags.geojson', { encoding: 'utf-8' })
16+
17+
const features: Array<Feature<Point, {
18+
name: string
19+
id: string
20+
}>> = []
21+
22+
for await (const doc of model.find({ 'metadata.leaf': true }).lean()) {
23+
const { metadata, area_name: areaName, pathTokens, ancestors } = doc
24+
25+
const ancestorArray = ancestors.split(',')
26+
const pointFeature = point(doc.metadata.lnglat.coordinates, {
27+
id: metadata.area_id.toUUID().toString(),
28+
name: areaName,
29+
type: 'crag',
30+
parent: {
31+
id: ancestorArray[ancestorArray.length - 2],
32+
name: pathTokens[doc.pathTokens.length - 2]
33+
}
34+
})
35+
features.push(pointFeature)
36+
}
37+
stream.write(JSON.stringify(featureCollection(features)) + os.EOL)
38+
stream.close()
39+
}
40+
41+
/**
42+
* Export crag groups as Geojson. Crag groups are immediate parent of leaf areas (crags/boulders).
43+
*/
44+
async function exportCragGroups (): Promise<void> {
45+
const model = getAreaModel()
46+
const stream = createWriteStream('crag-groups.geojson', { encoding: 'utf-8' })
47+
48+
interface CragGroup {
49+
uuid: MUUID
50+
name: string
51+
polygon: Polygon
52+
childAreaList: Array<{
53+
name: string
54+
uuid: MUUID
55+
leftRightIndex: number
56+
}>
57+
}
58+
59+
const rs: CragGroup[] = await model.aggregate([
60+
{ $match: { 'metadata.leaf': true } },
61+
{
62+
$lookup: {
63+
from: 'areas',
64+
localField: '_id',
65+
foreignField: 'children',
66+
as: 'parentCrags'
67+
}
68+
},
69+
{
70+
$match: {
71+
$and: [
72+
{ parentCrags: { $type: 'array', $ne: [] } }
73+
]
74+
}
75+
},
76+
{
77+
$unwind: '$parentCrags'
78+
},
79+
{
80+
$addFields: {
81+
parentCrags: {
82+
childId: '$metadata.area_id'
83+
}
84+
}
85+
},
86+
{
87+
$group: {
88+
_id: {
89+
uuid: '$parentCrags.metadata.area_id',
90+
name: '$parentCrags.area_name',
91+
polygon: '$parentCrags.metadata.polygon'
92+
},
93+
childAreaList: {
94+
$push: {
95+
leftRightIndex: '$metadata.leftRightIndex',
96+
uuid: '$metadata.area_id',
97+
name: '$area_name'
98+
}
99+
}
100+
}
101+
},
102+
{
103+
$project: {
104+
_id: 0,
105+
uuid: '$_id.uuid',
106+
name: '$_id.name',
107+
polygon: '$_id.polygon',
108+
childAreaList: 1
109+
}
110+
}
111+
])
112+
113+
const features: Array<Feature<Polygon, {
114+
name: string
115+
id: string
116+
}>> = []
117+
118+
for await (const doc of rs) {
119+
const polygonFeature = feature(doc.polygon, {
120+
type: 'crag-group',
121+
name: doc.name,
122+
id: doc.uuid.toUUID().toString(),
123+
children: doc.childAreaList.map(({ uuid, name, leftRightIndex }) => (
124+
{ id: uuid.toUUID().toString(), name, lr: leftRightIndex }))
125+
})
126+
features.push(polygonFeature)
127+
}
128+
129+
stream.write(JSON.stringify(featureCollection(features)) + os.EOL)
130+
stream.close()
131+
}
132+
133+
async function onDBConnected (): Promise<void> {
134+
logger.info('Start exporting crag data as Geojson')
135+
await exportLeafCrags()
136+
await exportCragGroups()
137+
await gracefulExit()
138+
}
139+
140+
void connectDB(onDBConnected)

0 commit comments

Comments
 (0)