#!/usr/bin/env bash

git submodule update

# Strip out empty lines and comments for conciseness:
yaml=`cat uap-core/regexes.yaml | sed '/\s*#/d' | sed '/^\s*$/d'`

# Build and format a Go file including our sources:
echo "package main
var UaparserDefinitionYaml = []byte(\`$yaml\`)" | gofmt > useragent-regexes-data.go

# изменение формата
yaml=`cat searchengine-and-social-list/SearchEngines.yml | sed '/\s*#/d' | sed '/^\s*$/d' | sed  -r 's/^ /     /'| sed  -r 's/^([^ ][^:]*).*/  - name: \1\n    items:/'`
echo "package main
var SearchEnginesDefinitionYaml = []byte(\`$yaml\`)" | gofmt > searchengine-data.go

# изменение формата
yaml=`cat searchengine-and-social-list/Socials.yml | sed '/\s*#/d' | sed '/^\s*$/d' | sed  -r 's/^ /     /'| sed  -r 's/^([^ ][^:]*).*/  - name: \1\n    urls:/'`
echo "package main
var SocialsDefinitionYaml = []byte(\`$yaml\`)" | gofmt > social-network-data.go


# top level domains
cat > domains.csv <<EOL
com
org
net
EOL

cat second-level-domains/SLDs.csv | sed -r 's/^.([^,]+).*/\1/' | sort | uniq >> domains.csv
cat "second-level-domains/TLDs without SLDs.csv" | sed 's/\.//' >> domains.csv
# доменны начинающиеся на com|org|net|co|it|edu
cat second-level-domains/SLDs.csv | sed -r 's/^[^,]+,\.?//' | grep -E '^(com|org|net|co|it|edu)\.' > domains.second.csv
# убрать лишние переносы и добавить кавычки
domains=`cat domains.csv | tr -d '\r'  | sed -r 's/(.+)/"\1",/' | tr -d '\n'`
domains_second=`cat domains.second.csv | tr -d '\r'  | sed -r 's/(.+)/"\1",/' | tr -d '\n'`
echo "package main
var SearchEngineTLD = []string{$domains}
var SearchEngineSLD = []string{$domains_second}" | gofmt > searchengine-tld-data.go
rm domains.csv
rm domains.second.csv