diff --git a/README.md b/README.md index cd9859e..1e6c924 100644 --- a/README.md +++ b/README.md @@ -28,3 +28,29 @@ func main() { // Output: 4 } ``` + +More Complex Usage +```go +import ( + "github.com/urturn/go-phantomjs" // exported package is phantomjs +) + +func main() { + p, err := phantomjs.Start() + if err != nil { + panic(err) + } + defer p.Exit() // Don't forget to kill phantomjs at some point. + var result interface{} + err = p.Run("function(done){ setTimeout(function() { done(3 + 3) ; }, 0);", &result) + if err != nil { + panic(err) + } + number, ok := result.(float64) + if !ok { + panic("Cannot convert result to float64") + } + fmt.Println(number) + // Output: 4 +} +``` diff --git a/phantom.go b/phantom.go index 0506146..e00da9a 100644 --- a/phantom.go +++ b/phantom.go @@ -4,36 +4,78 @@ import ( "bufio" "encoding/json" "errors" - "fmt" "io" "io/ioutil" + "log" "os" "os/exec" "strings" + "sync" + "time" ) +// Phantom a data structure that interacts with the wrapper file type Phantom struct { - cmd *exec.Cmd - in io.WriteCloser - out io.ReadCloser - errout io.ReadCloser + cmd *exec.Cmd + in io.WriteCloser + out io.ReadCloser + errout io.ReadCloser + readerErrorLock *sync.Mutex + readerLock *sync.Mutex + readerOut *bufio.Reader + readerErr *bufio.Reader + lineOut chan string + lineErr chan string + quit chan bool + stopReading bool + once *sync.Once + + nothingReadCount int64 +} + +// return Value is used to bundle up the value returned from the reader +type returnValue struct { + val string + err error +} + +// return Value is used to bundle up the value returned from the reader +type exitValue struct { + val []byte + err error } var nbInstance = 0 + var wrapperFileName = "" +var fileLock = new(sync.Mutex) + +var readerBufferSize = 2048 +var maxReadTimes = 100 /* +Start create phantomjs file Create a new `Phantomjs` instance and return it as a pointer. If an error occurs during command start, return it instead. */ + +var cmd = "phantomjs" + +// SetCommand lets you specify the binary for phantomjs +func SetCommand(cmd string) { + cmd = cmd +} + func Start(args ...string) (*Phantom, error) { + fileLock.Lock() if nbInstance == 0 { wrapperFileName, _ = createWrapperFile() } - nbInstance += 1 + nbInstance++ + fileLock.Unlock() args = append(args, wrapperFileName) - cmd := exec.Command("phantomjs", args...) + cmd := exec.Command(cmd, args...) inPipe, err := cmd.StdinPipe() if err != nil { @@ -51,100 +93,299 @@ func Start(args ...string) (*Phantom, error) { } p := Phantom{ - cmd: cmd, - in: inPipe, - out: outPipe, - errout: errPipe, + cmd: cmd, + in: inPipe, + out: outPipe, + errout: errPipe, + readerErrorLock: new(sync.Mutex), + readerLock: new(sync.Mutex), + lineOut: make(chan string, 100), + lineErr: make(chan string, 100), + quit: make(chan bool, 1), + stopReading: false, + once: new(sync.Once), + nothingReadCount: 0, } - err = cmd.Start() + p.readerOut = bufio.NewReaderSize(p.out, readerBufferSize*1024) + p.readerErr = bufio.NewReaderSize(p.errout, readerBufferSize*1024) + + go p.readFromSTDOut() + go p.readFromSTDERR() + err = cmd.Start() if err != nil { return nil, err } + // time.Sleep(time.Millisecond) return &p, nil } +func readreader(readerLock *sync.Mutex, reader *bufio.Reader) (string, error) { + var count = 0 + for { + readerLock.Lock() + data, _, err := reader.ReadLine() + readerLock.Unlock() + + if err != nil { + // Nothing else to read + if err == io.EOF && len(data) == 0 { + // like the scanner if 100 empty reads panic + count++ + if count >= 100 { + break + } + } + // Nothing to read and waiting to exit + if len(data) == 0 && strings.Contains(err.Error(), "bad file descriptor") { + return "", errors.New("Wrapper Error: Bad File Descriptor") + } else if err == io.EOF && len(data) == 0 { + return "", err + } + } + + line := string(data) + parts := strings.SplitN(line, " ", 2) + + if strings.HasPrefix(line, "RES") { + return parts[1], nil + } else if line != "" { + log.Printf("JS-LOG %s\n", line) + // return "", nil + } else if line != " " { + // return "", errors.New("Error reading response, just got a space") + } + } + return "", errors.New("EOF") +} + +func (p *Phantom) readFromSTDOut() { + for !p.stopReading { + line, err := readreader(p.readerLock, p.readerOut) + if err == io.EOF && line == "" { + // Done Reading Data + return + } else if err != nil && !p.stopReading { + // p.Exit() + log.Printf("An error occurred while reading stdout: %+v", err) + return + } else if p.stopReading && line == "" { + return + } else if line == "" { + continue + } + + select { + case p.lineOut <- line: + default: + log.Println("no listener attached to stdout " + line) + } + } +} + +func (p *Phantom) readFromSTDERR() { + for !p.stopReading { + line, err := readreader(p.readerErrorLock, p.readerErr) + if err == io.EOF && line == "" { + // Done Reading Data + return + } else if err != nil && !p.stopReading { + // p.Exit() + log.Printf("an error occurred while reading stderr: %+v", err) + return + } else if p.stopReading && line == "" { + return + } else if line == "" { + continue + } + + select { + case p.lineErr <- line: + default: + log.Println("no listener attached to sderr " + line) + } + } +} + +func drainBool(commch chan bool) { + for { + select { + case <-commch: + default: + return + } + } +} + /* Exit Phantomjs by sending the "phantomjs.exit()" command and wait for the command to end. -Return an error if one occured during exit command or if the program output a error value +Return an error if one occurred during exit command or if the program output a error value */ func (p *Phantom) Exit() error { - err := p.Load("phantom.exit()") - if err != nil { - return err + var err error + p.once.Do(func() { + err = p.Load("phantom.exit()") + if err != nil { + return + } + p.quit <- true + + p.readerErrorLock.Lock() + p.readerLock.Lock() + err = p.cmd.Wait() + p.readerLock.Unlock() + p.readerErrorLock.Unlock() + + fileLock.Lock() + nbInstance-- + if nbInstance == 0 { + err = os.Remove(wrapperFileName) + } + fileLock.Unlock() + }) + + return err +} + +/* +ForceShutdown will forcefully kill phantomjs. +This will completly terminate the proccess compared to Exit which will safely Exit +*/ +func (p *Phantom) ForceShutdown() error { + var err error + p.once.Do(func() { + err = p.Load("phantom.exit()") + if err != nil { + return + } + p.quit <- true + + p.readerErrorLock.Lock() + p.readerLock.Lock() + err = stopExec(p.cmd) + p.readerLock.Unlock() + p.readerErrorLock.Unlock() + + fileLock.Lock() + nbInstance-- + if nbInstance == 0 { + err = os.Remove(wrapperFileName) + } + fileLock.Unlock() + }) + + if !p.cmd.ProcessState.Exited() { + err = p.cmd.Process.Kill() + err = p.cmd.Process.Release() + + fileLock.Lock() + nbInstance-- + if nbInstance == 0 { + err = os.Remove(wrapperFileName) + } + fileLock.Unlock() } - err = p.cmd.Wait() - if err != nil { - return err + return err +} + +func stopExec(cmd *exec.Cmd) error { + done := make(chan error, 1) + go func() { + done <- cmd.Wait() + }() + select { + case <-time.After(3 * time.Second): + if err := cmd.Process.Kill(); err != nil { + errOther := cmd.Process.Release() + if errOther != nil { + log.Printf("Error not Handled %s", errOther) + } + return err + } + return nil + case err := <-done: + if err != nil { + return err + } + return nil } - nbInstance -= 1 - if nbInstance == 0 { - os.Remove(wrapperFileName) +} + +/* +SetMaxBufferSize will set the max Buffer Size +If your script will return a large input use this. +Specify the number of KB +Default value is 2048KB +*/ +func (p *Phantom) SetMaxBufferSize(bufferSize int) { + if bufferSize > 0 { + readerBufferSize = bufferSize } +} - return nil +/* +SetMaxReadTimes will set the max read times +If the +*/ +func (p *Phantom) SetMaxReadTimes(bufferSize int) { + if bufferSize > 0 { + readerBufferSize = bufferSize + } +} + +func drainchan(commch chan string) { + for { + select { + case <-commch: + default: + return + } + } } /* Run the javascript function passed as a string and wait for the result. -The result can be either in the return value of the function or the first argument passed -to the function first arguments. +The result can be either in the return value of the function or, +You can pass a function a closure which can take two arguments 1st is the successfull response +the 2nd is an err. See TestComplex in the phantom_test.go */ func (p *Phantom) Run(jsFunc string, res *interface{}) error { + if p.stopReading { + return errors.New("PhantomJS Instance is dead") + } + // flushing channel incase it read some left over data + drainchan(p.lineOut) + drainchan(p.lineErr) + // end flush err := p.sendLine("RUN", jsFunc, "END") if err != nil { return err } - scannerOut := bufio.NewScanner(p.out) - scannerErrorOut := bufio.NewScanner(p.errout) - resMsg := make(chan string) - errMsg := make(chan error) - go func() { - for scannerOut.Scan() { - line := scannerOut.Text() - parts := strings.SplitN(line, " ", 2) - if strings.HasPrefix(line, "RES") { - resMsg <- parts[1] - close(resMsg) - return - } else { - fmt.Printf("LOG %s\n", line) - } - } - }() - go func() { - for scannerErrorOut.Scan() { - line := scannerErrorOut.Text() - parts := strings.SplitN(line, " ", 2) - if strings.HasPrefix(line, "RES") { - errMsg <- errors.New(parts[1]) - close(errMsg) - return - } else { - fmt.Printf("LOG %s\n", line) - } - } - }() select { - case text := <-resMsg: + case text := <-p.lineOut: if res != nil { + err = json.Unmarshal([]byte(text), res) if err != nil { return err } } return nil - case err := <-errMsg: - return err + case errLine := <-p.lineErr: + return errors.New(errLine) + case <-p.quit: + return errors.New("PhantomJS Instance Killed") } + } /* +Load will load more code Eval `jsCode` in the main context. */ func (p *Phantom) Load(jsCode string) error { @@ -174,7 +415,7 @@ func (p *Phantom) sendLine(lines ...string) error { for _, l := range lines { _, err := io.WriteString(p.in, l+"\n") if err != nil { - return errors.New("Cannot Send: `" + l + "`") + return errors.New("Cannot Send: `" + l + "` " + "phantomjs instance might be dead") } } return nil diff --git a/phantom_test.go b/phantom_test.go index 6cf8452..768b493 100644 --- a/phantom_test.go +++ b/phantom_test.go @@ -1,7 +1,11 @@ package phantomjs import ( + "runtime" + "strings" + "sync" "testing" + "time" ) func TestStartStop(t *testing.T) { @@ -81,6 +85,253 @@ func TestDoubleErrorSendDontCrash(t *testing.T) { p.Run("function(done) {done(null, 'manual'); done(null, 'should not panic');}", nil) } +func TestThrow(t *testing.T) { + p, err := Start() + if err != nil { + panic(err) + } + defer p.Exit() // Don't forget to kill phantomjs at some point. + var result interface{} + err = p.Run("function() { throw 'Ooops' }", &result) + if err == nil { + t.Fatal("Expected an Error") + } + if !strings.Contains("\"Ooops\"", err.Error()) { + t.Fatal(err) + } +} + +func TestComplex(t *testing.T) { + var wg sync.WaitGroup + + for i := 0; i < 5; i++ { + wg.Add(1) + go func() { + defer wg.Done() + p, err := Start() + failOnError(err, t) + defer p.Exit() + var r interface{} + begin := time.Now() + + p.Run(`function(done){ + var a = 0; + var b = 1; + var c = 0; + for(var i=2; i<=25; i++) + { + c = b + a; + a = b; + b = c; + } + done(c, undefined); + }`, &r) + t.Logf("Completed Run in %s", time.Since(begin)) + failOnError(err, t) + v, ok := r.(float64) + if !ok { + t.Errorf("Should be an int but is %v", r) + return + } + if v != 75025 { + t.Errorf("Should be %d but is %f", 75025, v) + } + }() + } + wg.Wait() +} + +func TestForceShutdown(t *testing.T) { + p, err := Start() + failOnError(err, t) + + count := 0 + for i := 0; i < 5; i++ { + var r interface{} + + if i == 2 { + err = p.ForceShutdown() + failOnError(err, t) + } + err = p.Run(`function(done){ + var a = 0; + var b = 1; + var c = 0; + for(var i=2; i<=25; i++) + { + c = b + a; + a = b; + b = c; + } + done(c, undefined); + }`, &r) + if err == nil { + v, ok := r.(float64) + if !ok { + t.Errorf("Should be an int but is %v", r) + return + } + if v != 75025 { + t.Errorf("Should be %d but is %f", 75025, v) + } else { + count++ + } + } else { + if !strings.Contains(err.Error(), "PhantomJS Instance is dead") && !strings.Contains(err.Error(), "phantomjs instance might be dead") { + t.Fatal(err) + } + } + } + if count != 2 { + t.Fatalf("Didn't reach destination %d", count) + } + err = p.Exit() + failOnError(err, t) +} + +func TestRestartInstance(t *testing.T) { + p, err := Start() + defer p.Exit() + failOnError(err, t) + + err = p.ForceShutdown() + failOnError(err, t) + + p, err = Start() + defer p.Exit() + failOnError(err, t) + + var r interface{} + + err = p.Run(`function(done){ + var a = 0; + var b = 1; + var c = 0; + for(var i=2; i<=25; i++) + { + c = b + a; + a = b; + b = c; + } + done(c, undefined); + }`, &r) + + failOnError(err, t) + v, ok := r.(float64) + if !ok { + t.Errorf("Should be an int but is %v", r) + return + } + if v != 75025 { + t.Errorf("Should be %d but is %f", 75025, v) + } +} + +func TestMutlipleThreads(t *testing.T) { + + var wg sync.WaitGroup + var stats runtime.MemStats + for i := 0; i < 3; i++ { + wg.Add(1) + + runtime.ReadMemStats(&stats) + + t.Logf("MEM: %d, MALC: %d, GOTHREAD: %d, %s", (stats.Sys / 1000000), stats.Mallocs, runtime.NumGoroutine(), "Starting GO ROUTINE") + go func() { + var insideStats runtime.MemStats + runtime.ReadMemStats(&insideStats) + t.Logf("MEM: %d, MALC: %d, GOTHREAD: %d, %s", (insideStats.Sys / 1000000), insideStats.Mallocs, runtime.NumGoroutine(), "Start of Processor") + // Staring new Thread + p, err := Start() + t.Logf("MEM: %d, MALC: %d, GOTHREAD: %d, %s", (insideStats.Sys / 1000000), insideStats.Mallocs, runtime.NumGoroutine(), "Started Processor") + + failOnError(err, t) + var r interface{} + for j := 0; j < 5; j++ { + runtime.ReadMemStats(&insideStats) + t.Logf("MEM: %d, MALC: %d, GOTHREAD: %d, %s", (insideStats.Sys / 1000000), insideStats.Mallocs, runtime.NumGoroutine(), "Starting RUN && BEGIN OF LOOP") + err = p.Run(`function(done){ + var a = 0; + var b = 1; + var c = 0; + var page = new WebPage(); + page.open('http://charles.lescampeurs.org/'); + page.onLoadFinished = function(status) { + for(var i=2; i<=25; i++) + { + c = b + a; + a = b; + b = c; + } + done(c, undefined); + } + }`, &r) + t.Logf("MEM: %d, MALC: %d, GOTHREAD: %d, %s", (insideStats.Sys / 1000000), insideStats.Mallocs, runtime.NumGoroutine(), "Ending RUN") + + if err == nil { + v, ok := r.(float64) + if !ok { + t.Errorf("Should be an int but is %v", r) + p.Exit() + defer wg.Done() + return + } + if v != 75025 { + t.Errorf("Should be %d but is %f", 75025, v) + } + } else { + if !strings.Contains(err.Error(), "PhantomJS Instance is dead") && !strings.Contains(err.Error(), "phantomjs instance might be dead") { + t.Fatal(err) + } + } + runtime.ReadMemStats(&insideStats) + + t.Logf("MEM: %d, MALC: %d, GOTHREAD: %d, %s", (insideStats.Sys / 1000000), insideStats.Mallocs, runtime.NumGoroutine(), "Ending Loop \n ") + + } + p.Exit() + defer wg.Done() + t.Logf("MEM: %d, MALC: %d, GOTHREAD: %d, %s", (insideStats.Sys / 1000000), insideStats.Mallocs, runtime.NumGoroutine(), "Done with Processor") + //Thread Done + }() + runtime.ReadMemStats(&stats) + t.Logf("MEM: %d, MALC: %d, GOTHREAD: %d, %s", (stats.Sys / 1000000), stats.Mallocs, runtime.NumGoroutine(), "Started GO ROUTINE") + } + wg.Wait() + runtime.ReadMemStats(&stats) + t.Logf("MEM: %d, MALC: %d, GOTHREAD: %d, %s", (stats.Sys / 1000000), stats.Mallocs, runtime.NumGoroutine(), "Done") +} + +func TestMultipleLogs(t *testing.T) { + p, err := Start() + defer p.Exit() + failOnError(err, t) + var r interface{} + + err = p.Run(`function(done){ + var a = 0; + var b = 1; + var c = 0; + for(var i=2; i<=25; i++) + { + c = b + a; + a = b; + b = c; + console.log(c) + } + done(c, undefined); + }`, &r) + failOnError(err, t) + v, ok := r.(float64) + if !ok { + t.Errorf("Should be an int but is %v", r) + return + } + if v != 75025 { + t.Errorf("Should be %d but is %f", 75025, v) + } +} + func assertFloatResult(jsFunc string, expected float64, p *Phantom, t *testing.T) { var r interface{} err := p.Run(jsFunc, &r)