sercanto/infra
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

deploy search.nix-community.org

Browse source 
This is migrating search.nix.gcs.io with the code from
https://github.com/grahamc/network/tree/master/flexo/hound
This commit is contained in:
zimbatm 2020-10-06 10:49:10 +02:00
parent 18346ee38d
commit 06ff21ea37
No known key found for this signature in database
GPG key ID: 71BAF6D40C1D63D7
11 changed files with 639 additions and 19 deletions

12
build01/configuration.nix
View file

@ -28,6 +28,8 @@ in
../profiles/common.nix
../profiles/docker.nix
../services/hound
] ++ userImports;
# /boot is a mirror raid
@ -40,6 +42,12 @@ in
networking.usePredictableInterfaceNames = false;
networking.dhcpcd.enable = false;
networking.firewall = {
# for Nginx
allowedTCPPorts = [ 443 80 ];
};
systemd.network = {
enable = true;
networks."eth0".extraConfig = ''
@ -53,6 +61,10 @@ in
'';
};
# nginx is being used as the frontend HTTP server for all the services
# running on the box
services.nginx.enable = true;
services.cron.enable = true;
services.cron.systemCronJobs = [
# record that this machine is alive

31
build01/hydra.nix
View file

@ -66,10 +66,6 @@ in {
};
};
config = {
networking.firewall = {
allowedTCPPorts = [ 443 80 ];
};
nixpkgs.config = {
whitelistedLicenses = with lib.licenses; [
unfreeRedistributable
@ -81,21 +77,18 @@ in {
];
};
services.nginx = {
enable = true;
virtualHosts = {
"hydra.nix-community.org" = {
forceSSL = true;
enableACME = true;
locations."/" = {
proxyPass = "http://localhost:${toString(hydraPort)}";
extraConfig = ''
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
'';
};
services.nginx.virtualHosts = {
"hydra.nix-community.org" = {
forceSSL = true;
enableACME = true;
locations."/" = {
proxyPass = "http://localhost:${toString(hydraPort)}";
extraConfig = ''
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
'';
};
};
};

25
services/hound/0001-Fail-to-start-if-any-repos-fail-to-index.patch Normal file
View file

@ -0,0 +1,25 @@
From f976792420843bd079cea3aa85e70cc2cdbe98c2 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Sun, 8 Oct 2017 10:59:05 -0400
Subject: [PATCH] Fail to start if any repos fail to index
---
cmds/houndd/main.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/cmds/houndd/main.go b/cmds/houndd/main.go
index 60152ca..dfba0fc 100644
--- a/cmds/houndd/main.go
+++ b/cmds/houndd/main.go
@@ -132,7 +132,7 @@ func main() {
log.Panic(err)
}
if !ok {
- info_log.Println("Some repos failed to index, see output above")
+ info_log.Fatalf("Some repos failed to index, see output above")
} else {
info_log.Println("All indexes built!")
}
--
2.14.2

222
services/hound/0002-Custom-branch-specifier-PR-275.patch Normal file
View file

@ -0,0 +1,222 @@
From e2fe2b0e4720a74e38258f1bb6bd9ee746bdc6ee Mon Sep 17 00:00:00 2001
From: Paul Boutes <paul.boutes@gmail.com>
Date: Fri, 5 Jan 2018 01:52:46 +0100
Subject: [PATCH 1/3] feat(vcs/git): add ability to use custom branch from vcs
config
---
vcs/git.go | 29 +++++++++++++++++++++++++----
vcs/git_test.go | 38 ++++++++++++++++++++++++++++++++++++++
2 files changed, 63 insertions(+), 4 deletions(-)
create mode 100644 vcs/git_test.go
diff --git a/vcs/git.go b/vcs/git.go
index f8c6682..3626f53 100644
--- a/vcs/git.go
+++ b/vcs/git.go
@@ -8,6 +8,7 @@ import (
"os/exec"
"path/filepath"
"strings"
+ "encoding/json"
)
const defaultRef = "master"
@@ -16,10 +17,29 @@ func init() {
Register(newGit, "git")
}
-type GitDriver struct{}
+type GitDriver struct {
+ Ref string `json:"ref"`
+}
func newGit(b []byte) (Driver, error) {
- return &GitDriver{}, nil
+ d := &GitDriver{}
+ if e := getRef(b, d); e != nil {
+ return nil, e
+ }
+ return d, nil
+}
+
+func getRef(b []byte, d *GitDriver) error {
+ if b != nil {
+ if e := json.Unmarshal(b, d); e != nil {
+ return e
+ }
+ }
+ if d.Ref == "" {
+ d.Ref = defaultRef
+ return nil
+ }
+ return nil
}
func (g *GitDriver) HeadRev(dir string) (string, error) {
@@ -69,7 +89,7 @@ func (g *GitDriver) Pull(dir string) (string, error) {
"--no-tags",
"--depth", "1",
"origin",
- fmt.Sprintf("+%s:remotes/origin/%s", defaultRef, defaultRef)); err != nil {
+ fmt.Sprintf("+%s:remotes/origin/%s", g.Ref, g.Ref)); err != nil {
return "", err
}
@@ -77,7 +97,7 @@ func (g *GitDriver) Pull(dir string) (string, error) {
"git",
"reset",
"--hard",
- fmt.Sprintf("origin/%s", defaultRef)); err != nil {
+ fmt.Sprintf("origin/%s", g.Ref)); err != nil {
return "", err
}
@@ -90,6 +110,7 @@ func (g *GitDriver) Clone(dir, url string) (string, error) {
"git",
"clone",
"--depth", "1",
+ "--branch", g.Ref,
url,
rep)
cmd.Dir = par
diff --git a/vcs/git_test.go b/vcs/git_test.go
new file mode 100644
index 0000000..4aa69ad
--- /dev/null
+++ b/vcs/git_test.go
@@ -0,0 +1,38 @@
+package vcs
+
+import "testing"
+
+func TestGitConfigWithCustomRef(t *testing.T) {
+ cfg := `{"ref": "custom"}`
+ d, err := New("git", []byte(cfg))
+ if err != nil {
+ t.Fatal(err)
+ }
+ git := d.Driver.(*GitDriver)
+ if git.Ref != "custom" {
+ t.Fatalf("expected branch of \"custom\", got %s", git.Ref)
+ }
+}
+
+func TestGitConfigWithoutRef(t *testing.T) {
+ cfg := `{"option": "option"}`
+ d, err := New("git", []byte(cfg))
+ if err != nil {
+ t.Fatal(err)
+ }
+ git := d.Driver.(*GitDriver)
+ if git.Ref != "master" {
+ t.Fatalf("expected branch of \"master\", got %s", git.Ref)
+ }
+}
+
+func TestGitConfigWithoutAdditionalConfig(t *testing.T) {
+ d, err := New("git", nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ git := d.Driver.(*GitDriver)
+ if git.Ref != "master" {
+ t.Fatalf("expected branch of \"master\", got %s", git.Ref)
+ }
+}
From efafc6a34ab914e0988398abfa5b0e9eb56f54c5 Mon Sep 17 00:00:00 2001
From: Paul Boutes <paul.boutes@gmail.com>
Date: Tue, 9 Jan 2018 00:47:27 +0100
Subject: [PATCH 2/3] feat: default ref value for GitDriver struct
---
vcs/git.go | 18 +++++++-----------
1 file changed, 7 insertions(+), 11 deletions(-)
diff --git a/vcs/git.go b/vcs/git.go
index 3626f53..5b255e5 100644
--- a/vcs/git.go
+++ b/vcs/git.go
@@ -2,13 +2,13 @@ package vcs
import (
"bytes"
+ "encoding/json"
"fmt"
"io"
"log"
"os/exec"
"path/filepath"
"strings"
- "encoding/json"
)
const defaultRef = "master"
@@ -22,22 +22,18 @@ type GitDriver struct {
}
func newGit(b []byte) (Driver, error) {
- d := &GitDriver{}
- if e := getRef(b, d); e != nil {
+ d := &GitDriver{
+ Ref: defaultRef,
+ }
+ if e := setRefFromConfig(b, d); e != nil {
return nil, e
}
return d, nil
}
-func getRef(b []byte, d *GitDriver) error {
+func setRefFromConfig(b []byte, d *GitDriver) error {
if b != nil {
- if e := json.Unmarshal(b, d); e != nil {
- return e
- }
- }
- if d.Ref == "" {
- d.Ref = defaultRef
- return nil
+ return json.Unmarshal(b, d)
}
return nil
}
From 4faf7e7492763208d2337d0102c00f74c950c405 Mon Sep 17 00:00:00 2001
From: Paul Boutes <paul.boutes@gmail.com>
Date: Tue, 9 Jan 2018 01:03:08 +0100
Subject: [PATCH 3/3] feat: remove unecessary function
---
vcs/git.go | 14 ++++++--------
1 file changed, 6 insertions(+), 8 deletions(-)
diff --git a/vcs/git.go b/vcs/git.go
index 5b255e5..364e1d3 100644
--- a/vcs/git.go
+++ b/vcs/git.go
@@ -25,17 +25,15 @@ func newGit(b []byte) (Driver, error) {
d := &GitDriver{
Ref: defaultRef,
}
- if e := setRefFromConfig(b, d); e != nil {
- return nil, e
+
+ if b == nil {
+ return d, nil
}
- return d, nil
-}
-func setRefFromConfig(b []byte, d *GitDriver) error {
- if b != nil {
- return json.Unmarshal(b, d)
+ if e := json.Unmarshal(b, d); e != nil {
+ return nil, e
}
- return nil
+ return d, nil
}
func (g *GitDriver) HeadRev(dir string) (string, error) {

25
services/hound/0003-PR-275-p1-Replace-master-in-the-default-base-URL-with-a-rev.patch Normal file
View file

@ -0,0 +1,25 @@
From 6b4c532f46e88f93d3619f7e3c5983f573510714 Mon Sep 17 00:00:00 2001
From: Graham Christensen <graham@grahamc.com>
Date: Tue, 10 Jul 2018 12:55:59 -0400
Subject: [PATCH] Replace master in the default base URL with a rev
---
config/config.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/config/config.go b/config/config.go
index ebda60d..b4a29fb 100644
--- a/config/config.go
+++ b/config/config.go
@@ -13,7 +13,7 @@ const (
defaultPushEnabled = false
defaultPollEnabled = true
defaultVcs = "git"
- defaultBaseUrl = "{url}/blob/master/{path}{anchor}"
+ defaultBaseUrl = "{url}/blob/{rev}/{path}{anchor}"
defaultAnchor = "#L{line}"
)
--
2.16.4

8
services/hound/README.md Normal file
View file

@ -0,0 +1,8 @@
# search.nix-community.org
This service indexes the code in various repositories in the NixOS
organization and makes them available for quick querying.
URL: https://search.nix-community.org/
To extend the list of indexed repos, see the ./hound.json file.

29
services/hound/default.nix Normal file
View file

@ -0,0 +1,29 @@
{ pkgs, ... }:
{
services.nginx.virtualHosts."search.nix-community.org" = {
enableACME = true;
forceSSL = true;
locations = {
"=/open_search.xml".alias = "${./open-search.xml}";
"/".proxyPass = "http://127.0.0.1:6080/";
};
};
services.hound = {
enable = true;
listen = "127.0.0.1:6080";
config = builtins.readFile ./hound.json;
package = pkgs.hound.overrideAttrs (x: {
patches = [
./0001-Fail-to-start-if-any-repos-fail-to-index.patch
./0002-Custom-branch-specifier-PR-275.patch
./0003-PR-275-p1-Replace-master-in-the-default-base-URL-with-a-rev.patch
];
});
};
systemd.services.hound.serviceConfig = {
Restart = "always";
RestartSec = 5;
};
}

228
services/hound/hound.json Normal file
View file

@ -0,0 +1,228 @@
{
"dbpath": "/var/lib/hound/data",
"max-concurrent-indexers": 1,
"repos": {
"NixOS-.github": {
"url": "https://github.com/NixOS/.github.git",
"vcs-config": {
"ref": "master"
}
},
"NixOS-cabal2nix": {
"url": "https://github.com/NixOS/cabal2nix.git",
"vcs-config": {
"ref": "master"
}
},
"NixOS-flake-registry": {
"url": "https://github.com/NixOS/flake-registry.git",
"vcs-config": {
"ref": "master"
}
},
"NixOS-hydra": {
"url": "https://github.com/NixOS/hydra.git",
"vcs-config": {
"ref": "master"
}
},
"NixOS-hydra-ant-logger": {
"url": "https://github.com/NixOS/hydra-ant-logger.git",
"vcs-config": {
"ref": "master"
}
},
"NixOS-hydra-provisioner": {
"url": "https://github.com/NixOS/hydra-provisioner.git",
"vcs-config": {
"ref": "master"
}
},
"NixOS-mobile-nixos": {
"url": "https://github.com/NixOS/mobile-nixos.git",
"vcs-config": {
"ref": "master"
}
},
"NixOS-mobile-nixos-website": {
"url": "https://github.com/NixOS/mobile-nixos-website.git",
"vcs-config": {
"ref": "master"
}
},
"NixOS-mvn2nix-maven-plugin": {
"url": "https://github.com/NixOS/mvn2nix-maven-plugin.git",
"vcs-config": {
"ref": "master"
}
},
"NixOS-nix": {
"url": "https://github.com/NixOS/nix.git",
"vcs-config": {
"ref": "master"
}
},
"NixOS-nix-eclipse": {
"url": "https://github.com/NixOS/nix-eclipse.git",
"vcs-config": {
"ref": "master"
}
},
"NixOS-nix-idea": {
"url": "https://github.com/NixOS/nix-idea.git",
"vcs-config": {
"ref": "master"
}
},
"NixOS-nix-mode": {
"url": "https://github.com/NixOS/nix-mode.git",
"vcs-config": {
"ref": "master"
}
},
"NixOS-nix-pills": {
"url": "https://github.com/NixOS/nix-pills.git",
"vcs-config": {
"ref": "master"
}
},
"NixOS-nixops": {
"url": "https://github.com/NixOS/nixops.git",
"vcs-config": {
"ref": "master"
}
},
"NixOS-nixops-aws": {
"url": "https://github.com/NixOS/nixops-aws.git",
"vcs-config": {
"ref": "master"
}
},
"NixOS-nixops-hetzner": {
"url": "https://github.com/NixOS/nixops-hetzner.git",
"vcs-config": {
"ref": "master"
}
},
"NixOS-nixos-artwork": {
"url": "https://github.com/NixOS/nixos-artwork.git",
"vcs-config": {
"ref": "master"
}
},
"NixOS-nixos-channel-scripts": {
"url": "https://github.com/NixOS/nixos-channel-scripts.git",
"vcs-config": {
"ref": "master"
}
},
"NixOS-nixos-hardware": {
"url": "https://github.com/NixOS/nixos-hardware.git",
"vcs-config": {
"ref": "master"
}
},
"NixOS-nixos-homepage": {
"url": "https://github.com/NixOS/nixos-homepage.git",
"vcs-config": {
"ref": "master"
}
},
"NixOS-nixos-org-configurations": {
"url": "https://github.com/NixOS/nixos-org-configurations.git",
"vcs-config": {
"ref": "master"
}
},
"NixOS-nixos-planet": {
"url": "https://github.com/NixOS/nixos-planet.git",
"vcs-config": {
"ref": "master"
}
},
"NixOS-nixos-search": {
"url": "https://github.com/NixOS/nixos-search.git",
"vcs-config": {
"ref": "master"
}
},
"NixOS-nixos-weekly": {
"url": "https://github.com/NixOS/nixos-weekly.git",
"vcs-config": {
"ref": "master"
}
},
"NixOS-nixpart": {
"url": "https://github.com/NixOS/nixpart.git",
"vcs-config": {
"ref": "master"
}
},
"NixOS-nixpkgs": {
"url": "https://github.com/NixOS/nixpkgs.git",
"vcs-config": {
"ref": "master"
}
},
"NixOS-npm2nix": {
"url": "https://github.com/NixOS/npm2nix.git",
"vcs-config": {
"ref": "master"
}
},
"NixOS-ofborg": {
"url": "https://github.com/NixOS/ofborg.git",
"vcs-config": {
"ref": "released"
}
},
"NixOS-patchelf": {
"url": "https://github.com/NixOS/patchelf.git",
"vcs-config": {
"ref": "master"
}
},
"NixOS-rfc-steering-committee": {
"url": "https://github.com/NixOS/rfc-steering-committee.git",
"vcs-config": {
"ref": "master"
}
},
"NixOS-rfc39": {
"url": "https://github.com/NixOS/rfc39.git",
"vcs-config": {
"ref": "master"
}
},
"NixOS-rfcs": {
"url": "https://github.com/NixOS/rfcs.git",
"vcs-config": {
"ref": "master"
}
},
"NixOS-security": {
"url": "https://github.com/NixOS/security.git",
"vcs-config": {
"ref": "master"
}
},
"NixOS-snapd-nix-base": {
"url": "https://github.com/NixOS/snapd-nix-base.git",
"vcs-config": {
"ref": "master"
}
},
"NixOS-templates": {
"url": "https://github.com/NixOS/templates.git",
"vcs-config": {
"ref": "master"
}
},
"nixos-users-wiki-wiki": {
"url": "https://github.com/nixos-users/wiki.wiki.git",
"url-pattern": {
"base-url": "{url}/{path}"
}
}
}
}

10
services/hound/open-search.xml Normal file
View file

@ -0,0 +1,10 @@
<?xml version="1.0" encoding="UTF-8"?>
<OpenSearchDescription xmlns="http://a9.com/-/spec/opensearch/1.1/">
<ShortName>Hound</ShortName>
<Description>Search code with Hound</Description>
<Tags>Hound</Tags>
<Url type="text/html"
method="get"
template="https://search.nix-community.org/?q={searchTerms}" />
</OpenSearchDescription>

61
services/hound/update-hound.py Executable file
View file

@ -0,0 +1,61 @@
#!/usr/bin/env nix-shell
#!nix-shell -i python3 -p python3Packages.python -p python3Packages.requests
import requests
import json
from pprint import pprint
blacklist = [
'https://github.com/NixOS/nixos.git',
'https://github.com/NixOS/systemd.git',
'https://github.com/NixOS/docker.git',
'https://github.com/NixOS/nixpkgs-channels.git',
'https://github.com/NixOS/nixops-dashboard.git',
'https://github.com/NixOS/nixos-foundation.git',
];
def all_for_org(org, blacklist):
resp = {}
next_url = 'https://api.github.com/orgs/{}/repos'.format(org)
while next_url is not None:
repo_resp = requests.get(next_url)
if 'next' in repo_resp.links:
next_url = repo_resp.links['next']['url']
else:
next_url = None
repos = repo_resp.json()
resp.update({
"{}-{}".format(org, repo['name']): {
'url': repo['clone_url'],
'vcs-config': {
'ref': repo['default_branch']
}
}
for repo in repos
if repo['clone_url'] not in blacklist
})
return resp
repos = all_for_org('NixOS', blacklist)
repos['nixos-users-wiki-wiki'] = {
"url" : "https://github.com/nixos-users/wiki.wiki.git",
"url-pattern" : {
"base-url" : "{url}/{path}"
}
}
print(json.dumps(
{
"max-concurrent-indexers" : 1,
"dbpath" : "/var/lib/hound/data",
"repos": repos
},
indent=4,
sort_keys=True
))

7
terraform/cloudflare_nix-community_org.tf
View file

@ -23,6 +23,13 @@ resource "cloudflare_record" "nix-community-org-hydra-CNAME" {
type = "CNAME"
}
resource "cloudflare_record" "nix-community-org-search-CNAME" {
zone_id = local.nix_community_org_zone_id
name = "search"
value = "build01.nix-community.com"
type = "CNAME"
}
resource "cloudflare_record" "nix-community-org-apex-A" {
zone_id = local.nix_community_org_zone_id
name = "@"