Initial QSfera import
This commit is contained in:
+69
@@ -0,0 +1,69 @@
|
||||
/*
|
||||
Copyright 2017 Google Inc. All rights reserved.
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
/*
|
||||
Package tika provides a client and server for downloading, starting, and using Apache Tika's (http://tika.apache.org) Server.
|
||||
|
||||
Start with basic imports:
|
||||
|
||||
import "github.com/google/go-tika/tika"
|
||||
|
||||
You will need a running Server to make API calls to. So, if you don't
|
||||
have a server that is already running, and you don't have the Server
|
||||
JAR already downloaded, you can download one. The caller is responsible
|
||||
for removing the file when no longer needed.
|
||||
|
||||
Version is a custom type, and should be passed as such. There are constants in the code for these.
|
||||
The following example downloads version 1.21 to the named JAR in the
|
||||
current working directory.
|
||||
|
||||
err := tika.DownloadServer(context.Background(), tika.Version121, "tika-server-1.21.jar")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
If you don't have a running Tika Server, you can start one.
|
||||
|
||||
// Optionally pass a port as the second argument.
|
||||
s, err := tika.NewServer("tika-server-1.21.jar", "")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
err := s.Start(context.Background())
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer s.Stop()
|
||||
|
||||
To parse the contents of a file (or any io.Reader), you will need to open the io.Reader,
|
||||
create a client, and call client.Parse.
|
||||
|
||||
// import "os"
|
||||
f, err := os.Open("path/to/file")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
client := tika.NewClient(nil, s.URL())
|
||||
body, err := client.Parse(context.Background(), f)
|
||||
|
||||
If you pass an *http.Client to tika.NewClient, it will be used for all requests.
|
||||
|
||||
Some functions return a custom type, like Parsers(), Detectors(), and
|
||||
MIMETypes(). Use these to see what features are supported by the current
|
||||
Tika server.
|
||||
*/
|
||||
package tika
|
||||
+303
@@ -0,0 +1,303 @@
|
||||
/*
|
||||
Copyright 2017 Google Inc.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package tika
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha512"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"golang.org/x/net/context/ctxhttp"
|
||||
)
|
||||
|
||||
// Server represents a Tika server. Create a new Server with NewServer,
|
||||
// start it with Start, and shut it down with the close function returned
|
||||
// from Start.
|
||||
// There is no need to create a Server for an already running Tika Server
|
||||
// since you can pass its URL directly to a Client.
|
||||
// Additional Java system properties can be added to a Taka Server before
|
||||
// startup by adding to the JavaProps map
|
||||
type Server struct {
|
||||
jar string
|
||||
url string // url is derived from port.
|
||||
port string
|
||||
cmd *exec.Cmd
|
||||
child *ChildOptions
|
||||
JavaProps map[string]string
|
||||
}
|
||||
|
||||
// ChildOptions represent command line parameters that can be used when Tika is run with the -spawnChild option.
|
||||
// If a field is less than or equal to 0, the associated flag is not included.
|
||||
type ChildOptions struct {
|
||||
MaxFiles int
|
||||
TaskPulseMillis int
|
||||
TaskTimeoutMillis int
|
||||
PingPulseMillis int
|
||||
PingTimeoutMillis int
|
||||
}
|
||||
|
||||
func (co *ChildOptions) args() []string {
|
||||
if co == nil {
|
||||
return nil
|
||||
}
|
||||
args := []string{}
|
||||
args = append(args, "-spawnChild")
|
||||
if co.MaxFiles == -1 || co.MaxFiles > 0 {
|
||||
args = append(args, "-maxFiles", strconv.Itoa(co.MaxFiles))
|
||||
}
|
||||
if co.TaskPulseMillis > 0 {
|
||||
args = append(args, "-taskPulseMillis", strconv.Itoa(co.TaskPulseMillis))
|
||||
}
|
||||
if co.TaskTimeoutMillis > 0 {
|
||||
args = append(args, "-taskTimeoutMillis", strconv.Itoa(co.TaskTimeoutMillis))
|
||||
}
|
||||
if co.PingPulseMillis > 0 {
|
||||
args = append(args, "-pingPulseMillis", strconv.Itoa(co.PingPulseMillis))
|
||||
}
|
||||
if co.PingTimeoutMillis > 0 {
|
||||
args = append(args, "-pingTimeoutMillis", strconv.Itoa(co.PingTimeoutMillis))
|
||||
}
|
||||
return args
|
||||
}
|
||||
|
||||
// URL returns the URL of this Server.
|
||||
func (s *Server) URL() string {
|
||||
return s.url
|
||||
}
|
||||
|
||||
// NewServer creates a new Server. The default port is 9998.
|
||||
func NewServer(jar, port string) (*Server, error) {
|
||||
if jar == "" {
|
||||
return nil, fmt.Errorf("no jar file specified")
|
||||
}
|
||||
|
||||
// Check if the jar file exists.
|
||||
if _, err := os.Stat(jar); os.IsNotExist(err) {
|
||||
return nil, fmt.Errorf("jar file %q does not exist", jar)
|
||||
}
|
||||
|
||||
if port == "" {
|
||||
port = "9998"
|
||||
}
|
||||
|
||||
urlString := "http://localhost:" + port
|
||||
u, err := url.Parse(urlString)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid port %q: %v", port, err)
|
||||
}
|
||||
|
||||
s := &Server{
|
||||
jar: jar,
|
||||
port: port,
|
||||
url: u.String(),
|
||||
JavaProps: map[string]string{},
|
||||
}
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// ChildMode sets up the server to use the -spawnChild option.
|
||||
// If used, ChildMode must be called before starting the server.
|
||||
// If you want to turn off the -spawnChild option, call Server.ChildMode(nil).
|
||||
func (s *Server) ChildMode(ops *ChildOptions) error {
|
||||
if s.cmd != nil {
|
||||
return fmt.Errorf("server process already started, cannot switch to spawn child mode")
|
||||
}
|
||||
s.child = ops
|
||||
return nil
|
||||
}
|
||||
|
||||
var command = exec.Command
|
||||
|
||||
// Start starts the given server. Start will start a new Java process. The
|
||||
// caller must call Stop() to shut down the process when finished with the
|
||||
// Server. Start will wait for the server to be available or until ctx is
|
||||
// cancelled.
|
||||
func (s *Server) Start(ctx context.Context) error {
|
||||
if _, err := os.Stat(s.jar); os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
|
||||
// Create a slice of Java system properties to be passed to the JVM.
|
||||
props := []string{}
|
||||
for k, v := range s.JavaProps {
|
||||
props = append(props, fmt.Sprintf("-D%s=%q", k, v))
|
||||
}
|
||||
|
||||
args := append(append(props, "-jar", s.jar, "-p", s.port), s.child.args()...)
|
||||
cmd := command("java", args...)
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
return err
|
||||
}
|
||||
s.cmd = cmd
|
||||
|
||||
if err := s.waitForStart(ctx); err != nil {
|
||||
out, readErr := cmd.CombinedOutput()
|
||||
if readErr != nil {
|
||||
return fmt.Errorf("error reading output: %v", readErr)
|
||||
}
|
||||
// Report stderr since sometimes the server says why it failed to start.
|
||||
return fmt.Errorf("error starting server: %v\nserver stderr:\n\n%s", err, out)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// waitForServer waits until the given Server is responding to requests or
|
||||
// ctx is Done().
|
||||
func (s Server) waitForStart(ctx context.Context) error {
|
||||
c := NewClient(nil, s.url)
|
||||
t := time.NewTicker(500 * time.Millisecond)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-t.C:
|
||||
if _, err := c.Version(ctx); err == nil {
|
||||
return nil
|
||||
}
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Stop shuts the server down, killing the underlying Java process. Stop
|
||||
// must be called when finished with the server to avoid leaking the
|
||||
// Java process. If s has not been started, Stop will panic.
|
||||
// If not running in a Windows environment, it is recommended to use Shutdown
|
||||
// for a more graceful shutdown of the Java process.
|
||||
func (s *Server) Stop() error {
|
||||
if err := s.cmd.Process.Kill(); err != nil {
|
||||
return fmt.Errorf("could not kill server: %v", err)
|
||||
}
|
||||
if err := s.cmd.Wait(); err != nil {
|
||||
return fmt.Errorf("could not wait for server to finish: %v", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Shutdown attempts to close the server gracefully before using SIGKILL,
|
||||
// Stop() uses SIGKILL right away, which causes the kernal to stop the java process instantly.
|
||||
func (s *Server) Shutdown(ctx context.Context) error {
|
||||
if err := s.cmd.Process.Signal(os.Interrupt); err != nil {
|
||||
return fmt.Errorf("could not interrupt server: %v", err)
|
||||
}
|
||||
errChannel := make(chan error)
|
||||
go func() {
|
||||
select {
|
||||
case errChannel <- s.cmd.Wait():
|
||||
case <-ctx.Done():
|
||||
}
|
||||
}()
|
||||
select {
|
||||
case err := <-errChannel:
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not wait for server to finish: %v", err)
|
||||
}
|
||||
case <-ctx.Done():
|
||||
if err := s.cmd.Process.Kill(); err != nil {
|
||||
return fmt.Errorf("could not kill server: %v", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func sha512Hash(path string) (string, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
h := sha512.New()
|
||||
if _, err := io.Copy(h, f); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return fmt.Sprintf("%x", h.Sum(nil)), nil
|
||||
}
|
||||
|
||||
// A Version represents a Tika Server version.
|
||||
type Version string
|
||||
|
||||
// Supported versions of Tika Server.
|
||||
const (
|
||||
Version119 Version = "1.19"
|
||||
Version120 Version = "1.20"
|
||||
Version121 Version = "1.21"
|
||||
)
|
||||
|
||||
// Versions is a list of supported versions of Apache Tika.
|
||||
var Versions = []Version{Version119, Version120, Version121}
|
||||
|
||||
var sha512s = map[Version]string{
|
||||
Version119: "a9e2b6186cdb9872466d3eda791d0e1cd059da923035940d4b51bb1adc4a356670fde46995725844a2dd500a09f3a5631d0ca5fbc2d61a59e8e0bd95c9dfa6c2",
|
||||
Version120: "a7ef35317aba76be8606f9250893efece8b93384e835a18399da18a095b19a15af591e3997828d4ebd3023f21d5efad62a91918610c44e692cfd9bed01d68382",
|
||||
Version121: "e705c836b2110530c8d363d05da27f65c4f6c9051b660cefdae0e5113c365dbabed2aa1e4171c8e52dbe4cbaa085e3d8a01a5a731e344942c519b85836da646c",
|
||||
}
|
||||
|
||||
// DownloadServer downloads and validates the given server version,
|
||||
// saving it at path. DownloadServer returns an error if it could
|
||||
// not be downloaded/validated.
|
||||
// It is the caller's responsibility to remove the file when no longer needed.
|
||||
// If the file already exists and has the correct sha512, DownloadServer will
|
||||
// do nothing.
|
||||
func DownloadServer(ctx context.Context, v Version, path string) error {
|
||||
hash := sha512s[v]
|
||||
if hash == "" {
|
||||
return fmt.Errorf("unsupported Tika version: %s", v)
|
||||
}
|
||||
if got, err := sha512Hash(path); err == nil {
|
||||
if got == hash {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
out, err := os.Create(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error creating file: %v", err)
|
||||
}
|
||||
defer out.Close()
|
||||
|
||||
url := fmt.Sprintf("http://search.maven.org/remotecontent?filepath=org/apache/tika/tika-server/%s/tika-server-%s.jar", v, v)
|
||||
resp, err := ctxhttp.Get(ctx, nil, url)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to download %q: %v", url, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if _, err := io.Copy(out, resp.Body); err != nil {
|
||||
return fmt.Errorf("error saving download: %v", err)
|
||||
}
|
||||
|
||||
h, err := sha512Hash(path)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if h != hash {
|
||||
if err := os.Remove(path); err != nil {
|
||||
return fmt.Errorf("invalid sha512: %s: error removing %s: %v", h, path, err)
|
||||
}
|
||||
return fmt.Errorf("invalid sha512: %s", h)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
+357
@@ -0,0 +1,357 @@
|
||||
/*
|
||||
Copyright 2017 Google Inc.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package tika
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"reflect"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ClientError is returned by Client's various parse methods and
|
||||
// represents an error response from the Tika server. Example usage:
|
||||
//
|
||||
// client := tika.NewClient(nil, tikaURL)
|
||||
// s, err := client.Parse(context.Background(), input)
|
||||
// var tikaErr tika.ClientError
|
||||
// if errors.As(err, &tikaErr) {
|
||||
// switch tikaErr.StatusCode {
|
||||
// case http.StatusUnsupportedMediaType, http.StatusUnprocessableEntity:
|
||||
// // Handle content related error
|
||||
// default:
|
||||
// // Handle possibly intermittent http error
|
||||
// }
|
||||
// } else if err != nil {
|
||||
// // Handle non-http error
|
||||
// }
|
||||
type ClientError struct {
|
||||
// StatusCode is the HTTP status code returned by the Tika server.
|
||||
StatusCode int
|
||||
}
|
||||
|
||||
func (e ClientError) Error() string {
|
||||
return fmt.Sprintf("response code %d", e.StatusCode)
|
||||
}
|
||||
|
||||
// Client represents a connection to a Tika Server.
|
||||
type Client struct {
|
||||
// url is the URL of the Tika Server, including the port (if necessary), but
|
||||
// not the trailing slash. For example, http://localhost:9998.
|
||||
url string
|
||||
// HTTPClient is the client that will be used to call the Tika Server. If no
|
||||
// client is specified, a default client will be used. Since http.Clients are
|
||||
// thread safe, the same client will be used for all requests by this Client.
|
||||
httpClient *http.Client
|
||||
}
|
||||
|
||||
// NewClient creates a new Client. If httpClient is nil, the http.DefaultClient will be
|
||||
// used.
|
||||
func NewClient(httpClient *http.Client, urlString string) *Client {
|
||||
return &Client{httpClient: httpClient, url: urlString}
|
||||
}
|
||||
|
||||
// A Parser represents a Tika Parser. To get a list of all Parsers, see Parsers().
|
||||
type Parser struct {
|
||||
Name string
|
||||
Decorated bool
|
||||
Composite bool
|
||||
Children []Parser
|
||||
SupportedTypes []string
|
||||
}
|
||||
|
||||
// MIMEType represents a Tika MIME Type. To get a list of all MIME Types, see
|
||||
// MIMETypes.
|
||||
type MIMEType struct {
|
||||
Alias []string
|
||||
SuperType string
|
||||
}
|
||||
|
||||
// A Detector represents a Tika Detector. Detectors are used to get the filetype
|
||||
// of a file. To get a list of all Detectors, see Detectors().
|
||||
type Detector struct {
|
||||
Name string
|
||||
Composite bool
|
||||
Children []Detector
|
||||
}
|
||||
|
||||
// Translator represents the Java package of a Tika Translator.
|
||||
type Translator string
|
||||
|
||||
// Translators available by default in Tika. You must configure all required
|
||||
// authentication details in Tika Server (for example, an API key).
|
||||
const (
|
||||
Lingo24Translator Translator = "org.apache.tika.language.translate.Lingo24Translator"
|
||||
GoogleTranslator Translator = "org.apache.tika.language.translate.GoogleTranslator"
|
||||
MosesTranslator Translator = "org.apache.tika.language.translate.MosesTranslator"
|
||||
JoshuaTranslator Translator = "org.apache.tika.language.translate.JoshuaTranslator"
|
||||
MicrosoftTranslator Translator = "org.apache.tika.language.translate.MicrosoftTranslator"
|
||||
YandexTranslator Translator = "org.apache.tika.language.translate.YandexTranslator"
|
||||
)
|
||||
|
||||
// XTIKAContent is the metadata field of the content of a file after recursive
|
||||
// parsing. See ParseRecursive and MetaRecursive.
|
||||
const XTIKAContent = "X-TIKA:content"
|
||||
|
||||
// call makes the given request to c and returns the response body.
|
||||
// call returns an error and a nil reader if the response code is not 200 StatusOK.
|
||||
func (c *Client) call(ctx context.Context, input io.Reader, method, path string, header http.Header) (io.ReadCloser, error) {
|
||||
if c.httpClient == nil {
|
||||
c.httpClient = http.DefaultClient
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, method, c.url+path, input)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header = header
|
||||
|
||||
resp, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
resp.Body.Close()
|
||||
return nil, ClientError{resp.StatusCode}
|
||||
}
|
||||
return resp.Body, nil
|
||||
}
|
||||
|
||||
// callString makes the given request to c and returns the result as a string
|
||||
// and error. callString returns an error if the response code is not 200 StatusOK.
|
||||
func (c *Client) callString(ctx context.Context, input io.Reader, method, path string, header http.Header) (string, error) {
|
||||
body, err := c.call(ctx, input, method, path, header)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer body.Close()
|
||||
|
||||
b := &strings.Builder{}
|
||||
if _, err := io.Copy(b, body); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return b.String(), nil
|
||||
}
|
||||
|
||||
// Parse parses the given input, returning the body of the input as a string and an error.
|
||||
// If the error is not nil, the body is undefined.
|
||||
func (c *Client) Parse(ctx context.Context, input io.Reader) (string, error) {
|
||||
return c.ParseWithHeader(ctx, input, nil)
|
||||
}
|
||||
|
||||
// ParseReader parses the given input, returning the body of the input as a reader and an error.
|
||||
// If the error is nil, the returned reader must be closed, else, the reader is nil.
|
||||
func (c *Client) ParseReader(ctx context.Context, input io.Reader) (io.ReadCloser, error) {
|
||||
return c.ParseReaderWithHeader(ctx, input, nil)
|
||||
}
|
||||
|
||||
// ParseWithHeader parses the given input, returning the body of the input as a string and an error.
|
||||
// If the error is not nil. the body is undefined.
|
||||
// This function also accepts a header so the caller can specify things like `Accept`
|
||||
func (c *Client) ParseWithHeader(ctx context.Context, input io.Reader, header http.Header) (string, error) {
|
||||
return c.callString(ctx, input, "PUT", "/tika", header)
|
||||
}
|
||||
|
||||
// ParseReaderWithHeader parses the given input, returning the body of the input as a reader and an error.
|
||||
// If the error is nil, the returned reader must be closed, else, the reader is nil.
|
||||
// This function also accepts a header so the caller can specify things like `Accept`
|
||||
func (c *Client) ParseReaderWithHeader(ctx context.Context, input io.Reader, header http.Header) (io.ReadCloser, error) {
|
||||
return c.call(ctx, input, "PUT", "/tika", header)
|
||||
}
|
||||
|
||||
// ParseRecursive parses the given input and all embedded documents, returning a
|
||||
// list of the contents of the input with one element per document. See
|
||||
// MetaRecursive for access to all metadata fields. If the error is not nil, the
|
||||
// result is undefined.
|
||||
func (c *Client) ParseRecursive(ctx context.Context, input io.Reader) ([]string, error) {
|
||||
m, err := c.MetaRecursive(ctx, input)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var r []string
|
||||
for _, d := range m {
|
||||
if content := d[XTIKAContent]; len(content) > 0 {
|
||||
r = append(r, content[0])
|
||||
}
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// Meta parses the metadata from the given input, returning the metadata and an
|
||||
// error. If the error is not nil, the metadata is undefined.
|
||||
func (c *Client) Meta(ctx context.Context, input io.Reader) (string, error) {
|
||||
return c.MetaWithHeader(ctx, input, nil)
|
||||
}
|
||||
|
||||
// MetaWithHeader parses the metadata from the given input, returning the metadata and an
|
||||
// error. If the error is not nil, the metadata is undefined.
|
||||
// This function also accepts a header so the caller can specify things like `Accept`
|
||||
func (c *Client) MetaWithHeader(ctx context.Context, input io.Reader, header http.Header) (string, error) {
|
||||
return c.callString(ctx, input, "PUT", "/meta", header)
|
||||
}
|
||||
|
||||
// MetaField parses the metadata from the given input and returns the given
|
||||
// field. If the error is not nil, the result string is undefined.
|
||||
func (c *Client) MetaField(ctx context.Context, input io.Reader, field string) (string, error) {
|
||||
return c.MetaFieldWithHeader(ctx, input, field, nil)
|
||||
}
|
||||
|
||||
// MetaFieldWithHeader parses the metadata from the given input and returns the given
|
||||
// field. If the error is not nil, the result string is undefined.
|
||||
// This function also accepts a header so the caller can specify things like `Accept`
|
||||
func (c *Client) MetaFieldWithHeader(ctx context.Context, input io.Reader, field string, header http.Header) (string, error) {
|
||||
return c.callString(ctx, input, "PUT", fmt.Sprintf("/meta/%v", field), header)
|
||||
}
|
||||
|
||||
// Detect gets the mimetype of the given input, returning the mimetype and an
|
||||
// error. If the error is not nil, the mimetype is undefined.
|
||||
func (c *Client) Detect(ctx context.Context, input io.Reader) (string, error) {
|
||||
return c.callString(ctx, input, "PUT", "/detect/stream", nil)
|
||||
}
|
||||
|
||||
// Language detects the language of the given input, returning the two letter
|
||||
// language code and an error. If the error is not nil, the language is
|
||||
// undefined.
|
||||
func (c *Client) Language(ctx context.Context, input io.Reader) (string, error) {
|
||||
return c.callString(ctx, input, "PUT", "/language/stream", nil)
|
||||
}
|
||||
|
||||
// LanguageString detects the language of the given string, returning the two letter
|
||||
// language code and an error. If the error is not nil, the language is
|
||||
// undefined.
|
||||
func (c *Client) LanguageString(ctx context.Context, input string) (string, error) {
|
||||
r := strings.NewReader(input)
|
||||
return c.callString(ctx, r, "PUT", "/language/string", nil)
|
||||
}
|
||||
|
||||
// MetaRecursive parses the given input and all embedded documents. The result
|
||||
// is a list of maps from metadata key to value for each document. The content
|
||||
// of each document is in the XTIKAContent field in text form. See
|
||||
// ParseRecursive to just get the content of each document. If the error is not
|
||||
// nil, the result list is undefined.
|
||||
func (c *Client) MetaRecursive(ctx context.Context, input io.Reader) ([]map[string][]string, error) {
|
||||
return c.MetaRecursiveType(ctx, input, "text")
|
||||
}
|
||||
|
||||
// MetaRecursiveType parses the given input and all embedded documents. The result
|
||||
// is a list of maps from metadata key to value for each document. The content
|
||||
// of each document is in the XTIKAContent field, and is of the type indicated
|
||||
// by the contentType parameter An empty string can be passed in for a default
|
||||
// type of XML. See ParseRecursive to just get the content of each document. If
|
||||
// the error is not nil, the result list is undefined.
|
||||
func (c *Client) MetaRecursiveType(ctx context.Context, input io.Reader, contentType string) ([]map[string][]string, error) {
|
||||
path := "/rmeta"
|
||||
if contentType != "" {
|
||||
path = fmt.Sprintf("/rmeta/%s", contentType)
|
||||
}
|
||||
body, err := c.call(ctx, input, "PUT", path, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer body.Close()
|
||||
var m []map[string]interface{}
|
||||
if err := json.NewDecoder(body).Decode(&m); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var r []map[string][]string
|
||||
for _, d := range m {
|
||||
doc := make(map[string][]string)
|
||||
r = append(r, doc)
|
||||
for k, v := range d {
|
||||
switch vt := v.(type) {
|
||||
case string:
|
||||
doc[k] = []string{vt}
|
||||
case []interface{}:
|
||||
for _, i := range vt {
|
||||
s, ok := i.(string)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("field %q has value %v and type %T, expected a string or []string", k, v, vt)
|
||||
}
|
||||
doc[k] = append(doc[k], s)
|
||||
}
|
||||
default:
|
||||
return nil, fmt.Errorf("field %q has value %v and type %v, expected a string or []string", k, v, reflect.TypeOf(v))
|
||||
}
|
||||
}
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// Translate returns an error and the translated input from src language to
|
||||
// dst language using t. If the error is not nil, the translation is undefined.
|
||||
func (c *Client) Translate(ctx context.Context, input io.Reader, t Translator, src, dst string) (string, error) {
|
||||
return c.callString(ctx, input, "POST", fmt.Sprintf("/translate/all/%s/%s/%s", t, src, dst), nil)
|
||||
}
|
||||
|
||||
// TranslateReader translates the given input from src language to dst language using t.
|
||||
// It returns the translated document as a reader. If an error occurs, the reader is nil, else, the reader
|
||||
// must be closed by the caller after usage.
|
||||
func (c *Client) TranslateReader(ctx context.Context, input io.Reader, t Translator, src, dst string) (io.ReadCloser, error) {
|
||||
return c.call(ctx, input, "POST", fmt.Sprintf("/translate/all/%s/%s/%s", t, src, dst), nil)
|
||||
}
|
||||
|
||||
// Version returns the default hello message from Tika server.
|
||||
func (c *Client) Version(ctx context.Context) (string, error) {
|
||||
return c.callString(ctx, nil, "GET", "/version", nil)
|
||||
}
|
||||
|
||||
var jsonHeader = http.Header{"Accept": []string{"application/json"}}
|
||||
|
||||
// callUnmarshal is like call, but unmarshals the JSON response into v.
|
||||
func (c *Client) callUnmarshal(ctx context.Context, path string, v interface{}) error {
|
||||
body, err := c.call(ctx, nil, "GET", path, jsonHeader)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer body.Close()
|
||||
return json.NewDecoder(body).Decode(v)
|
||||
}
|
||||
|
||||
// Parsers returns the list of available parsers and an error. If the error is
|
||||
// not nil, the list is undefined. To get all available parsers, iterate through
|
||||
// the Children of every Parser.
|
||||
func (c *Client) Parsers(ctx context.Context) (*Parser, error) {
|
||||
p := new(Parser)
|
||||
if err := c.callUnmarshal(ctx, "/parsers/details", p); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
|
||||
// MIMETypes returns a map from MIME Type name to MIMEType, or properties about
|
||||
// that specific MIMEType.
|
||||
func (c *Client) MIMETypes(ctx context.Context) (map[string]MIMEType, error) {
|
||||
mt := make(map[string]MIMEType)
|
||||
if err := c.callUnmarshal(ctx, "/mime-types", &mt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return mt, nil
|
||||
}
|
||||
|
||||
// Detectors returns the list of available Detectors for this server. To get all
|
||||
// available detectors, iterate through the Children of every Detector.
|
||||
func (c *Client) Detectors(ctx context.Context) (*Detector, error) {
|
||||
d := new(Detector)
|
||||
if err := c.callUnmarshal(ctx, "/detectors", d); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
Reference in New Issue
Block a user