Skip to content

Commit

Permalink
Restructured ingest & search
Browse files Browse the repository at this point in the history
  • Loading branch information
JeffSpies committed Oct 5, 2020
1 parent cc3c828 commit d53cf92
Show file tree
Hide file tree
Showing 17 changed files with 1,242 additions and 98 deletions.
1 change: 1 addition & 0 deletions client/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
"node-sass": "^4.13.1",
"quasar": "^1.14.0",
"react": "^16.13.1",
"typesense": "^0.7.1",
"v-runtime-template": "^1.10.0",
"validator": "^13.0.0",
"vue-apollo": "^3.0.2",
Expand Down
64 changes: 31 additions & 33 deletions client/src/pages/pages/Search.vue
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
<q-input
tabindex="-1"
v-model="search"
debounce="500"
debounce="100"
filled
square
placeholder="Search"
Expand Down Expand Up @@ -84,8 +84,9 @@

<script>
import _ from 'lodash'
import { Client } from 'typesense'
import { mapActions } from 'vuex'
// import { mapActions } from 'vuex'
// import ProjectCard from '../../components/search/ProjectCard'
import ProfileCard from '../../components/search/ProfileCard'
Expand All @@ -101,25 +102,36 @@ export default {
data: [],
page: 1,
cardPerPage: 10,
loading: false
loading: false,
client: null
}
},
components: {
// ProjectCard,
ProfileCard
},
mounted () {
this.search = this.$route.query.q
// this.search = this.$route.query.q
this.client = new Client({
'nodes': [{
'host': 'localhost',
'path': '/typesense',
'port': '8000',
'protocol': 'http'
}],
'apiKey': 'e5325d85-7570-4d95-b95d-60c99cfba4bf',
'connectionTimeoutSeconds': 2
})
},
watch: {
async search () {
await this.esSearch()
await this.doSearch()
// Note: No need to push the search as a query string
// It will call the backend after each event in search input
this.$router.push({
path: 'search',
query: { ...this.$route.query, q: this.search }
})
// this.$router.push({
// path: 'search',
// query: { ...this.$route.query, q: this.search }
// })
},
startDate () {
this.$router.push({
Expand All @@ -146,30 +158,16 @@ export default {
}
},
methods: {
...mapActions('app', ['search']),
async esSearch () {
try {
this.data = []
this.loading = true
if (_.isEmpty(this.search)) return
const { data } = await this.$axios(
{
url: '/search',
method: 'POST',
data: { search: this.search, types: [ 'users' ] }
}
)
if (!_.isEmpty(data)) {
this.data = data['users']
}
} catch (error) {
console.error(error.message)
} finally {
this.loading = false
}
// ...mapActions('app', ['search']),
async doSearch () {
const { hits } = await this.client
.collections(['databrary-users'])
.documents()
.search({
q: this.search,
query_by: 'familyName,givenName,additionalName,displayFullName,bio'
})
this.data = _.map(hits, 'document')
}
}
}
Expand Down
19 changes: 17 additions & 2 deletions client/yarn.lock
Original file line number Diff line number Diff line change
Expand Up @@ -2172,6 +2172,13 @@ axios@^0.18.1:
follow-redirects "1.5.10"
is-buffer "^2.0.2"

axios@^0.20.0:
version "0.20.0"
resolved "https://registry.yarnpkg.com/axios/-/axios-0.20.0.tgz#057ba30f04884694993a8cd07fa394cff11c50bd"
integrity sha512-ANA4rr2BDcmmAQLOKft2fufrtuvlqR+cXNNinUmvfeSNCOF98PZL+7M/v1zIdGo7OLjEA9J2gXJL+j4zGsl0bA==
dependencies:
follow-redirects "^1.10.0"

babel-eslint@^10.0.3:
version "10.1.0"
resolved "https://registry.yarnpkg.com/babel-eslint/-/babel-eslint-10.1.0.tgz#6968e568a910b78fb3779cdd8b6ac2f479943232"
Expand Down Expand Up @@ -4861,7 +4868,7 @@ follow-redirects@1.5.10:
dependencies:
debug "=3.1.0"

follow-redirects@^1.0.0:
follow-redirects@^1.0.0, follow-redirects@^1.10.0:
version "1.13.0"
resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.13.0.tgz#b42e8d93a2a7eea5ed88633676d6597bc8e384db"
integrity sha512-aq6gF1BEKje4a9i9+5jimNFIpq4Q1WiwBToeRK5NvZBd/TRsmW8BsJfOEGkr76TbOyPVD3OVDN910EcUNtRYEA==
Expand Down Expand Up @@ -6674,7 +6681,7 @@ log-update@4.0.0:
slice-ansi "^4.0.0"
wrap-ansi "^6.2.0"

loglevel@^1.6.8:
loglevel@^1.6.8, loglevel@^1.7.0:
version "1.7.0"
resolved "https://registry.yarnpkg.com/loglevel/-/loglevel-1.7.0.tgz#728166855a740d59d38db01cf46f042caa041bb0"
integrity sha512-i2sY04nal5jDcagM3FMfG++T69GEEM8CYuOfeOIvmXzOIcwE9a/CJPR0MFM97pYMj/u10lzz7/zd7+qwhrBTqQ==
Expand Down Expand Up @@ -10344,6 +10351,14 @@ typescript@^3.7.5:
resolved "https://registry.yarnpkg.com/typescript/-/typescript-3.9.7.tgz#98d600a5ebdc38f40cb277522f12dc800e9e25fa"
integrity sha512-BLbiRkiBzAwsjut4x/dsibSTB6yWpwT5qWmC2OfuCg3GgVQCSgMs4vEctYPhsaGtd0AeuuHMkjZ2h2WG8MSzRw==

typesense@^0.7.1:
version "0.7.1"
resolved "https://registry.yarnpkg.com/typesense/-/typesense-0.7.1.tgz#304e0f22dc9e06a2c0cc5f8539492d1faf9f98ba"
integrity sha512-ZoROcThAJJyPpCzNEWQu8ve3JkWCow4VIyEYE4KzuaRmNrqmvuDwImK7BSs5O16uMHsF7a1gQj/XbbUpERp2xA==
dependencies:
axios "^0.20.0"
loglevel "^1.7.0"

uglify-js@^3.1.4, uglify-js@^3.5.1:
version "3.10.4"
resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-3.10.4.tgz#dd680f5687bc0d7a93b14a3482d16db6eba2bfbb"
Expand Down
7 changes: 6 additions & 1 deletion docker-assets/nginx/default.conf.template
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,12 @@ server {
# rewrite ^/hasura/(.*)$ $1 break;
}

location ~ /(login|register|search|session|logout|keycloak|minio|hasura|ingest) {
location /typesense/ {
proxy_pass http://${DOCKER_HOST_IP}:8108/;
proxy_set_header "X-TYPESENSE-API-KEY" "e5325d85-7570-4d95-b95d-60c99cfba4bf";
}

location ~ /(login|register|index|session|logout|keycloak|minio|hasura|ingest) {
proxy_pass http://${DOCKER_HOST_IP}:${NEST_PORT};
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
Expand Down
2 changes: 2 additions & 0 deletions gql/registerUser.gql
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
mutation registerUser(
$id: Int,
$authServerId: String!,
$emailPrimary: String!,
$givenName: String!,
Expand All @@ -12,6 +13,7 @@ mutation registerUser(
$bio: String
) {
insert_users(objects: {
id: $id,
authServerId: $authServerId,
emailPrimary: $emailPrimary,
givenName: $givenName,
Expand Down
61 changes: 61 additions & 0 deletions ingests/20201004-lego.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
import _ from 'lodash'
import Knex from 'knex'
import pMap from 'p-map'
import {promises as fs} from 'fs'

import { bootstrap } from '../server-nest/dist/sdk'
import { KeycloakService } from '../server-nest/dist/keycloak/keycloak.service'
import { UserService } from '../server-nest/dist/users/user.service'
import { UserDTO } from '../server-nest/dist/dtos/user.dto'

async function main () {
const app = await bootstrap()
const keycloakService = app.get(KeycloakService)
const userService = app.get(UserService)

const knex = Knex({
client: 'pg',
connection: {
host : 'localhost',
port: 5432,
user : 'postgres',
password : 'postgres',
database : 'postgres'
}
})

const ingestData = JSON.parse(await fs.readFile('../data/ingests/20201004-lego.json', 'utf-8'))
const project = _.first(ingestData)

const successes = await pMap(_.get(project, 'access', []), async user => {
const party = _.get(user, 'party')
const newUser = {
id: party.id,
displayFullName: `${party.prename} ${party.sortname}`,
givenName: party.prename,
familyName: party.sortname,
emailPrimary: party.email,
authServerId: null
}
newUser.authServerId = await keycloakService.registerUser(
newUser.emailPrimary,
newUser.givenName,
newUser.familyName
)
const userDTO: UserDTO = new UserDTO(newUser)
try {
await userService.createUser(userDTO)
return true
} catch (error) {
console.log(`Error when adding ${newUser.emailPrimary} (${newUser.authServerId}): ${error.msg}`)
return false
}
})

console.log(`Added ${_.sum(successes)} users`)

await knex.destroy()
await app.close()
}

main()
5 changes: 5 additions & 0 deletions ingests/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,14 @@
"main": "index.js",
"license": "MIT",
"dependencies": {
"@types/lodash": "^4.14.161",
"@types/node": "^14.11.2",
"knex": "^0.21.6",
"lodash": "^4.17.20",
"meilisearch": "^0.14.0",
"moby": "^1.1.2",
"p-map": "^4.0.0",
"pg": "^8.4.0",
"rake-js": "^0.1.1"
}
}
46 changes: 46 additions & 0 deletions ingests/tsconfig.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
// {
// "compilerOptions": {
// "module": "commonjs",
// "declaration": true,
// "removeComments": true,
// "emitDecoratorMetadata": true,
// "experimentalDecorators": true,
// "target": "es2017",
// "sourceMap": true,
// "outDir": "./dist",
// "baseUrl": "./",
// "incremental": true,
// "esModuleInterop": true,
// },
// "exclude": ["node_modules", "dist"]
// }

{
"$schema": "https://json.schemastore.org/tsconfig",
"display": "Node 12",

"compilerOptions": {
"lib": ["es6"],
"allowJs": true,
"module": "commonjs",
"target": "esnext",
"outDir": "dist",
"strict": false,
"esModuleInterop": true,
"skipLibCheck": true,
"forceConsistentCasingInFileNames": true,
"moduleResolution": "node",
"sourceMap": true,
"experimentalDecorators": true
},

"include": [
"src/**/*",
"runs/**/*"
],
"exclude": [
"node_modules",
"dist",
"**/*.spec.ts"
]
}

0 comments on commit d53cf92

Please sign in to comment.