Skip to content

Your first source type ​

A working source type, from start to finish: it lists the files named in a JSON index on a web server, and downloads the ones the user imports.

It is written in Go because the app starts a program — source-type.exe on Windows — and Go makes one file with nothing to install. Any language that can make a program works the same way.

The index it reads ​

The user gives each source the address of a file like this:

json
[
  { "id": "ep-12", "title": "Episode 12", "url": "https://example.com/ep12.mp3", "mime": "audio/mpeg" },
  { "id": "notes", "title": "Show notes", "url": "https://example.com/notes.txt", "mime": "text/plain" }
]

What it answers ​

  1. getSourceTypeInfo — its name, and one setting: the index address. The app builds the "add a source" form from it.
  2. setOptions — fetches the index. If it cannot, it says so next to the field.
  3. getItems — lists every file at the top level. Audio and video get the stt cap, so the app knows to transcribe them; text files get text.
  4. getItem and getCaptions — one item; no subtitle tracks.
  5. download — fetches the file and sends it to the sink, then answers with a file name.
  6. ping and shutdown.

Anything else is answered with -32601, and every content call made before setOptions succeeded gets -32005.

The program ​

go
// Command webindex is an Orbuzo source type: it lists the files named in a JSON index on a web
// server, and downloads the ones the user imports.
package main

import (
	"bufio"
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"os"
	"strings"
	"sync"
	"time"
)

// A file in the index: [{"id": "ep-12", "title": "Episode 12", "url": "https://…", "mime": "audio/mpeg"}]
type entry struct {
	ID    string `json:"id"`
	Title string `json:"title"`
	URL   string `json:"url"`
	Mime  string `json:"mime"`
}

type request struct {
	ID     json.RawMessage `json:"id"`
	Method string          `json:"method"`
	Params json.RawMessage `json:"params"`
}

type rpcError struct {
	Code    int    `json:"code"`
	Message string `json:"message"`
	Data    any    `json:"data,omitempty"`
}

var (
	outMu   sync.Mutex
	mu      sync.Mutex
	entries map[string]entry // nil until setOptions succeeds
	order   []string
)

func main() {
	in := bufio.NewScanner(os.Stdin)
	in.Buffer(make([]byte, 64*1024), 1<<20) // one message is at most 1 MiB
	for in.Scan() {
		var req request
		if err := json.Unmarshal(in.Bytes(), &req); err != nil {
			logf("not JSON: %v", err)
			continue
		}
		if req.ID == nil {
			continue // a notification, like $/cancelRequest: this example ignores them
		}
		if req.Method == "shutdown" {
			reply(req.ID, nil, nil)
			os.Exit(0)
		}
		go func() { // requests can overlap: answer each in its own goroutine
			result, err := handle(req)
			reply(req.ID, result, err)
		}()
	}
	os.Exit(0) // stdin closed: the host is gone
}

func handle(req request) (any, *rpcError) {
	switch req.Method {
	case "getSourceTypeInfo":
		return map[string]any{
			"versions": map[string]string{"protocol": "1.0", "source_type": "0.1.0"},
			"about": map[string]string{
				"id":          "example.webindex",
				"name":        "Web index",
				"author":      "You",
				"description": "Files listed in a JSON index on a web server.",
			},
			"features": []string{},
			"options": []map[string]any{
				{"name": "index_url", "label": "Index address", "type": "string", "required": true},
			},
		}, nil
	case "setOptions":
		return setOptions(req.Params)
	case "ping":
		if loaded() {
			return map[string]string{"status": "ready"}, nil
		}
		return map[string]string{"status": "not_configured"}, nil
	}

	if !loaded() {
		return nil, &rpcError{Code: -32005, Message: "This source is not set up yet."}
	}
	var p struct {
		ID       string          `json:"id"`
		ParentID *string         `json:"parent_id"`
		Sink     json.RawMessage `json:"sink"`
	}
	_ = json.Unmarshal(req.Params, &p)

	switch req.Method {
	case "getItems":
		if p.ParentID != nil {
			return nil, &rpcError{Code: -32002, Message: "This folder does not exist."}
		}
		mu.Lock()
		defer mu.Unlock()
		items := make([]any, 0, len(order))
		for _, id := range order {
			items = append(items, item(entries[id]))
		}
		return map[string]any{"items": items, "next_cursor": nil}, nil
	case "getItem":
		e, ok := find(p.ID)
		if !ok {
			return nil, &rpcError{Code: -32002, Message: "This file is no longer in the index."}
		}
		return item(e), nil
	case "getCaptions":
		return map[string]any{"tracks": []any{}}, nil
	case "download":
		e, ok := find(p.ID)
		if !ok {
			return nil, &rpcError{Code: -32002, Message: "This file is no longer in the index."}
		}
		return download(e, p.Sink)
	}
	return nil, &rpcError{Code: -32601, Message: "Unknown method."}
}

func setOptions(params json.RawMessage) (any, *rpcError) {
	var opts struct {
		IndexURL string `json:"index_url"`
	}
	_ = json.Unmarshal(params, &opts)
	bad := func(msg string) *rpcError {
		return &rpcError{Code: -32602, Message: "Some settings need fixing.",
			Data: map[string]any{"fields": map[string]string{"index_url": msg}}}
	}
	if !strings.HasPrefix(opts.IndexURL, "https://") && !strings.HasPrefix(opts.IndexURL, "http://") {
		return nil, bad("Use an http:// or https:// address.")
	}
	client := http.Client{Timeout: 20 * time.Second}
	resp, err := client.Get(opts.IndexURL)
	if err != nil {
		return nil, bad("This address does not answer.")
	}
	defer resp.Body.Close()
	var list []entry
	if resp.StatusCode != http.StatusOK || json.NewDecoder(resp.Body).Decode(&list) != nil {
		return nil, bad("This address does not hold a file index.")
	}
	mu.Lock()
	defer mu.Unlock()
	entries = map[string]entry{}
	order = nil
	for _, e := range list {
		entries[e.ID] = e
		order = append(order, e.ID)
	}
	return map[string]string{"status": "ready"}, nil
}

// download fetches the file and sends it to the host's sink with one PUT. Only then does it answer.
func download(e entry, raw json.RawMessage) (any, *rpcError) {
	var sink struct {
		Kind    string            `json:"kind"`
		URL     string            `json:"url"`
		Headers map[string]string `json:"headers"`
	}
	_ = json.Unmarshal(raw, &sink)
	if u, err := url.Parse(sink.URL); sink.Kind != "http" || err != nil ||
		(u.Hostname() != "127.0.0.1" && u.Hostname() != "::1") {
		return nil, &rpcError{Code: -32602, Message: "The app asked for a download this source type cannot send."}
	}

	src, err := http.Get(e.URL)
	if err != nil {
		return nil, &rpcError{Code: -32004, Message: "The server is not answering. Try again later."}
	}
	defer src.Body.Close()
	if src.StatusCode != http.StatusOK {
		return nil, &rpcError{Code: -32003, Message: "The server would not send this file."}
	}

	put, _ := http.NewRequest(http.MethodPut, sink.URL, src.Body)
	put.ContentLength = src.ContentLength // -1 when unknown: then it is sent chunked
	for k, v := range sink.Headers {
		put.Header.Set(k, v)
	}
	put.Header.Set("Content-Type", e.Mime)
	resp, err := http.DefaultClient.Do(put)
	if err != nil {
		return nil, &rpcError{Code: -32603, Message: "The file could not be handed to Orbuzo."}
	}
	resp.Body.Close()
	if resp.StatusCode != http.StatusCreated {
		return nil, &rpcError{Code: -32603, Message: fmt.Sprintf("Orbuzo did not take the file (%d).", resp.StatusCode)}
	}
	return map[string]any{"file_name": e.Title + extension(e.Mime), "mime": e.Mime}, nil
}

func item(e entry) map[string]any {
	it := map[string]any{"id": e.ID, "parent_id": nil, "label": e.Title, "type": "file", "mime": e.Mime}
	switch {
	case strings.HasPrefix(e.Mime, "audio/"), strings.HasPrefix(e.Mime, "video/"):
		it["caps"] = []string{"stt"}
	case e.Mime == "text/plain":
		it["caps"] = []string{"text"}
	}
	return it
}

func extension(mime string) string {
	switch mime {
	case "audio/mpeg":
		return ".mp3"
	case "audio/mp4":
		return ".m4a"
	case "video/mp4":
		return ".mp4"
	case "text/plain":
		return ".txt"
	}
	return ""
}

func find(id string) (entry, bool) {
	mu.Lock()
	defer mu.Unlock()
	e, ok := entries[id]
	return e, ok
}

func loaded() bool {
	mu.Lock()
	defer mu.Unlock()
	return entries != nil
}

func reply(id json.RawMessage, result any, err *rpcError) {
	msg := map[string]any{"jsonrpc": "2.0", "id": id}
	if err != nil {
		msg["error"] = err
	} else {
		msg["result"] = result
	}
	line, _ := json.Marshal(msg)
	outMu.Lock()
	defer outMu.Unlock()
	os.Stdout.Write(append(line, '\n'))
}

// logf writes to stderr, which the app keeps in a log file. Never log addresses, tokens or titles.
func logf(format string, args ...any) {
	fmt.Fprintf(os.Stderr, "webindex: "+format+"\n", args...)
}

Build it ​

sh
go mod init example.com/webindex
GOOS=windows GOARCH=amd64 go build -o source-type.exe .

Try it ​

Put it in a folder of its own inside source-types/, next to Orbuzo.exe:

text
Orbuzo.exe
source-types/
  webindex/
    source-type.exe

Start Orbuzo, choose Add a source…, pick Web index, and paste your index address. Your files show up under the new source, marked STT or TXT. Right-click one and choose Import….

Install and test has more on finding problems.

What to add next ​

  • Paging. Return at most limit items, with a next_cursor for the rest.
  • Folders. Items with "type": "folder", and getItems for each parent_id.
  • Subtitles. List tracks in getCaptions, give files the cc cap, and send a track when download names it. Captions import in seconds and cost the user nothing.
  • Progress. Send progress notifications during a long download.
  • Cancelling. Stop a download on $/cancelRequest and answer -32800.