scrapligo -- is a Go library focused on connecting to devices, specifically network devices (routers/switches/firewalls/etc.) via SSH and NETCONF.

Hello.

Doing my first steps in Go while trying to implement Go agalogue of:

from ncclient import manager

with manager.connect(
    host="10.10.30.4",
    port=22,
    username="admin",
    password="Juniper",
) as m:
    r = m.get_config("running")
    print(r)

which works perfectly fine.

This is the code:

package main

import (
       "fmt"

       "github.com/scrapli/scrapligo/driver/base"
       "github.com/scrapli/scrapligo/netconf"
)

func getConfig(addr string) {
       d, _ := netconf.NewNetconfDriver(
               addr,
               base.WithPort(22),
               base.WithAuthStrictKey(false),
               base.WithAuthUsername("admin"),
               base.WithAuthPassword("Juniper"),
       )

       err := d.Open()
       if err != nil {
               fmt.Printf("failed to open driver; error: %+v\n", err)
               return
       }
       defer d.Close()

       r, err := d.GetConfig("running")
       if err != nil {
               fmt.Printf("failed to get config; error: %+v\n", err)
               return
       }

       fmt.Printf("Get Config Response:\n%s\n", r.Result)
}

func getRPC(addr string, rpc string) {
       d, _ := netconf.NewNetconfDriver(
               addr,
               base.WithPort(22),
               base.WithAuthStrictKey(false),
               base.WithAuthUsername("admin"),
               base.WithAuthPassword("Juniper"),
       )

       err := d.Open()
       if err != nil {
               fmt.Printf("failed to open driver; error: %+v\n", err)
               return
       }
       defer d.Close()

       r, err := d.RPC(netconf.WithNetconfFilter(rpc))
       if err != nil {
               fmt.Printf("failed to get config; error: %+v\n", err)
               return
       }

       fmt.Printf("Get Config Response:\n%s\n", r.Result)
}


func main() {
       getRPC("10.10.30.4", "<get-route-engine-information/>")
       getConfig("10.10.30.4")
}

RPC works but GetConfig doesn't:

Get Config Response:
<nc:rpc-reply xmlns:nc="urn:ietf:params:xml:ns:netconf:base:1.0" xmlns:junos="http://xml.juniper.net/junos/19.2R0/junos" xmlns="urn:ietf:params:xml:ns:netconf:base:1.0" message-id="101">
<route-engine-information xmlns="http://xml.juniper.net/junos/19.2R0/junos-chassis">
<route-engine>
<slot>0</slot>
<mastership-state>master</mastership-state>
<mastership-priority>master (default)</mastership-priority>
<status>OK</status>
<memory-dram-size>2002 MB</memory-dram-size>
<memory-installed-size>(2048 MB installed)</memory-installed-size>
<memory-buffer-utilization>45</memory-buffer-utilization>
<cpu-user>1</cpu-user>
<cpu-background>0</cpu-background>
<cpu-system>2</cpu-system>
<cpu-interrupt>1</cpu-interrupt>
<cpu-idle>96</cpu-idle>
<cpu-user1>2</cpu-user1>
<cpu-background1>0</cpu-background1>
<cpu-system1>2</cpu-system1>
<cpu-interrupt1>0</cpu-interrupt1>
<cpu-idle1>95</cpu-idle1>
<cpu-user2>2</cpu-user2>
<cpu-background2>0</cpu-background2>
<cpu-system2>2</cpu-system2>
<cpu-interrupt2>1</cpu-interrupt2>
<cpu-idle2>95</cpu-idle2>
<cpu-user3>2</cpu-user3>
<cpu-background3>0</cpu-background3>
<cpu-system3>3</cpu-system3>
<cpu-interrupt3>1</cpu-interrupt3>
<cpu-idle3>84</cpu-idle3>
<model>RE-VMX</model>
<start-time junos:seconds="1651771969">2022-05-05 17:32:49 UTC</start-time>
<up-time junos:seconds="2112">35 minutes, 12 seconds</up-time>
<last-reboot-reason>Router rebooted after a normal shutdown.</last-reboot-reason>
<load-average-one>0.44</load-average-one>
<load-average-five>0.42</load-average-five>
<load-average-fifteen>0.45</load-average-fifteen>
</route-engine>
</route-engine-information>
</nc:rpc-reply>
]]>]]>

Get Config Response:
<nc:rpc-reply xmlns:nc="urn:ietf:params:xml:ns:netconf:base:1.0" xmlns:junos="http://xml.juniper.net/junos/19.2R0/junos" xmlns="urn:ietf:params:xml:ns:netconf:base:1.0" message-id="101">
<nc:rpc-error>
<nc:error-type>protocol</nc:error-type>
<nc:error-tag>operation-failed</nc:error-tag>
<nc:error-severity>error</nc:error-severity>
<nc:error-message>syntax error, expecting &lt;candidate/&gt; or &lt;running/&gt;</nc:error-message>
<nc:error-info>
<nc:bad-element>running</nc:bad-element>
</nc:error-info>
</nc:rpc-error>
<nc:rpc-error>
<nc:error-type>protocol</nc:error-type>
<nc:error-tag>operation-failed</nc:error-tag>
<nc:error-severity>error</nc:error-severity>
<nc:error-message>syntax error, expecting &lt;candidate/&gt; or &lt;running/&gt;</nc:error-message>
<nc:error-info>
<nc:bad-element>running</nc:bad-element>
</nc:error-info>
</nc:rpc-error>
</nc:rpc-reply>
]]>]]>

What am I doing wrong here?

And there is one more thing: if I change port to 830 (works great from shell or ncclient) I get:

failed to open driver; error: error reading from transport, cannot continue

Could you pls shed some light on how to use that?! Thank you.

Hey Carl,

i'm trying to access an old Cisco C3560. I get the following error message. I´ve seen that there is no way to set the kexalgorithms.

failed to open driver; error: ssh: handshake failed: ssh: no common algorithm for key exchange; client offered: [[email protected] ecdh-sha2-nistp256 ecdh-sha2-nistp384 ecdh-sha2-nistp521 diffie-hellman-group14-sha1], server offered: [diffie-hellman-group1-sha1]

I´ve lookup in the ssh package, that you can set the kexalgorithms up: https://pkg.go.dev/golang.org/x/crypto/ssh#Config --> KeyExchanges []string

Im more then happy to test a new build, if we can come up with a solution :)

Regards, Romario

When using scrapligo from goroutines, go detects a race:

go run -race main.go
==================
WARNING: DATA RACE
Write at 0x000000a5f190 by goroutine 9:
  github.com/scrapli/scrapligo/channel.getAuthPatterns()
      /home/mk/go/pkg/mod/github.com/scrapli/[email protected]/channel/authenticate.go:21 +0x144
  github.com/scrapli/scrapligo/channel.(*Channel).authenticateSSH()
      /home/mk/go/pkg/mod/github.com/scrapli/[email protected]/channel/authenticate.go:34 +0x294
  github.com/scrapli/scrapligo/channel.(*Channel).AuthenticateSSH.func1()
      /home/mk/go/pkg/mod/github.com/scrapli/[email protected]/channel/authenticate.go:120 +0xc4

Previous read at 0x000000a5f190 by goroutine 10:
  github.com/scrapli/scrapligo/channel.getAuthPatterns()
      /home/mk/go/pkg/mod/github.com/scrapli/[email protected]/channel/authenticate.go:20 +0x30
  github.com/scrapli/scrapligo/channel.(*Channel).authenticateSSH()
      /home/mk/go/pkg/mod/github.com/scrapli/[email protected]/channel/authenticate.go:34 +0x294
  github.com/scrapli/scrapligo/channel.(*Channel).AuthenticateSSH.func1()
      /home/mk/go/pkg/mod/github.com/scrapli/[email protected]/channel/authenticate.go:120 +0xc4

Goroutine 9 (running) created at:
  github.com/scrapli/scrapligo/channel.(*Channel).AuthenticateSSH()
      /home/mk/go/pkg/mod/github.com/scrapli/scr[email protected]/channel/authenticate.go:119 +0x1ea
  github.com/scrapli/scrapligo/driver/base.(*Driver).Open()
      /home/mk/go/pkg/mod/github.com/scrapli/[email protected]/driver/base/base.go:74 +0x631
  github.com/scrapli/scrapligo/driver/network.(*Driver).Open()
      /home/mk/go/pkg/mod/github.com/scrapli/[email protected]/driver/network/network.go:82 +0x32
  main.getVersion()
      /home/mk/Network-Automation-with-Go/ch03/concurrency4/main.go:48 +0x544
  main.main·dwrap·4()
      /home/mk/Network-Automation-with-Go/ch03/concurrency4/main.go:114 +0x109

Can I suggest replacing

func getAuthPatterns() *authPatterns {
	if authPatternsInstance == nil {
		authPatternsInstance = &authPatterns{
			telnetLoginPattern: regexp.MustCompile(`(?im)^(.*username:)|(.*login:)\s?$`),
			passwordPattern:    regexp.MustCompile(`(?im)^(.*@.*)?password:\s?$`),
			passphrasePattern:  regexp.MustCompile(`(?i)enter passphrase for key`),
		}
	}

	return authPatternsInstance
}

with

func init() {
	if authPatternsInstance == nil {
		authPatternsInstance = &authPatterns{
			telnetLoginPattern: regexp.MustCompile(`(?im)^(.*username:)|(.*login:)\s?$`),
			passwordPattern:    regexp.MustCompile(`(?im)^(.*@.*)?password:\s?$`),
			passphrasePattern:  regexp.MustCompile(`(?i)enter passphrase for key`),
		}
	}

}

with these changes race is not detected.

When connecting to a Cisco IOSXE device (have not tested with others) that contains a login banner, where the login banner does not end on a blank line, the SSH Authentication times out as the passwordPattern in func GetAuthPatterns() is not matched

Example with IOSv that ships with Cisco CML, out of the box, the default login banner configuration is this:

banner login ^C
**************************************************************************
* IOSv is strictly limited to use for evaluation, demonstration and IOS  *
* education. IOSv is provided as-is and is not supported by Cisco's      *
* Technical Advisory Center. Any use or disclosure, in whole or in part, *
* of the IOSv Software or Documentation to any third party for any       *
* purposes is expressly prohibited except as otherwise authorized by     *
* Cisco in writing.                                                      *
**************************************************************************^C

because the banner does not end on a blank line the password prompt appear in a format that does not match the regular expression for the passwordPattern defined in func GetAuthPatterns(): passwordPattern: regexp.MustCompile(`(?im)^(.*@.*)?password:\s?$`),

removing the banner, or Changing the banner as follows does solve this problem:

banner login ^C
**************************************************************************
* IOSv is strictly limited to use for evaluation, demonstration and IOS  *
* education. IOSv is provided as-is and is not supported by Cisco's      *
* Technical Advisory Center. Any use or disclosure, in whole or in part, *
* of the IOSv Software or Documentation to any third party for any       *
* purposes is expressly prohibited except as otherwise authorized by     *
* Cisco in writing.                                                      *
**************************************************************************
^C

However, could this also be solved by changing the password prompt regex? Changing the pattern to: passwordPattern: regexp.MustCompile(`(?im)^(.*)?password:\s?$`), does solve the problem, but not sure if that will cause other issues (if there was use cases that required the .*@.* to match instead of just .*)?

alternatively, is there the ability to pass a custom regex for this prompt in anywhere?

Hello,

I have an issue while reading a device config, using a simple GetConfig("running"): the output returned by Result is truncated. Looking further, It's truncated here:

<next-hops>
         <next-hop>
          <index

The original data returned by the device is visible in the logs:

      <next-hops>
         <next-hop>
          <index>##10.222.18.65##</index>

We can see that the output is truncated at the ## character.

I think the issue lies with the record1dot1() function (since the router is using netconf 1.1), In particular,

chunkSections := patterns.v1dot1Chunk.FindAllSubmatch(r.RawResult, -1)

is cutting the final chunk at the <index> tag.

I can see with the debugger that the final chunk has length 45100, but it's size should be 106234. This sets the r.Failed (which I didn't notice before investigating this issue).

I therefore think that the v1dot1Chunk is wrong. A chunk on my device looks like this:

#106234
  <netconf-yang xmlns="http://cisco.com/ns/yang/Cisco-IOS-XR-um-netconf-yang-cfg">
   <agent>
    <ssh/>
   </agent>
  </netconf-yang>

and the pattern is v1dot1Chunk: regexp.MustCompile(`(?ms)(\d+)\n(.*?)#`),.

I had success on this router by changing the pattern to (?ms)(\d+)\n(.*?)^#, but I don't know if it breaks other things.

Some nxos devices require 2 XML namespace attributes in the RPC element. ( see https://community.cisco.com/t5/network-management/netconf-get-config-error-wrong-document-namespaces-not-specified/td-p/3298323 ) It would be nice if we could define a second namespace while doing netconf operations with such devices.

According to my testing, defining the namespace in the raw filter query does not work. After the request, I'll get back a default hello reply and the device closes the connection. Sometimes, there is an RPC namespace missing error msg.

Request:

2022/03/09 15:19:05 write::localhost::10022::write: 
<?xml version="1.0" encoding="UTF-8"?>
<rpc xmlns="urn:ietf:params:xml:ns:netconf:base:1.0" message-id="101">
<get>
<filter type="subtree">
<show xmlns="http://www.cisco.com/nxos:1.0:nfcli">
<interface>
</interface>
</show>
</filter>
</get>
</rpc>]]>]]>

Reply

2022/03/09 15:19:05 debug::localhost::10022::read: <?xml version="1.0" encoding="utf-8"?>
<hello xmlns="urn:ietf:params:xml:ns:netconf:base:1.0">
     <capabilities>
         <capability>urn:ietf:params:netconf:base:1.0</capability>
     </capabilities>
</hello>]]>]]>

Reply with error msg

022/03/09 15:19:03 debug::localhost::10022::read: <?xml version="1.0" encoding="ISO-8859-1"?>
<rpc-reply xmlns="urn:ietf:params:xml:ns:netconf:base:1.0" message-id="101">
  <rpc-error>
    <error-type>rpc</error-type>
    <error-tag>missing-attribute</error-tag>
    <error-severity>error</error-severity>
    <error-message>Wrong document: namespaces not specified</error-message>
    <error-info>
      <bad-element>rpc</bad-element>
    </error-info>
  </rpc-error>
</rpc-reply>
]]>]]>

Hello,

While testing a juniper device (finding the issue requiring to pass options.WithNetconfForceSelfClosingTags(),).

I get the following r.Result (only the end, for brevity):

</configuration>
</nc:data>
</nc:rpc-reply>
]]>]]>

(note that there is not newline after the prompt)

It seems to me that this shouldn't be the case. During my other tests, the output was only the XML. One of the difference between my previous tests is that this device only supports netconf 1.0 (I guess there is something different with the 1.0 code path).

here is my sample code :

package main import ( "fmt" "github.com/scrapli/scrapligo/response" "github.com/scrapli/scrapligo/driver/options" "github.com/scrapli/scrapligo/platform" "time" )

func main() { for { _, err := ExecCmd([]string{"sho run","show ver"}, "x.x.x.x", "cml20") if err != nil { panic("wrong") } time.Sleep(10 * time.Duration(time.Second)) }

}

func ExecCmd(cmdlist []string, deviceip string, devicename string) (response.MultiResponse, error) { p, err := platform.NewPlatform( // cisco_iosxe refers to the included cisco iosxe platform definition "cisco_iosxr", deviceip, options.WithPort(22), options.WithAuthNoStrictKey(), options.WithTimeoutOps(120time.Second), options.WithAuthUsername("admin"), options.WithAuthPassword("admin"), options.WithTermWidth(256), options.WithTermHeight(0), ) if err != nil { fmt.Printf("failed to create platform for %s(%s); error: %+v\n", devicename, deviceip, err) return nil, err } d, err := p.GetNetworkDriver() if err != nil { fmt.Printf("failed to fetch network driver for %s(%s) ; error: %+v\n", devicename, deviceip, err) return nil, err } err = d.Open() if err != nil { fmt.Printf("failed to open driver %s(%s); error: %+v\n", devicename, deviceip, err) return nil, err } defer d.Close() mr, err := d.SendCommands(cmdlist, options.WithTimeoutOps(120*time.Second)) return mr, err }

I found lots of defunct processes, Every 1.0s: ps -ef | grep ssh | grep -v grep | grep -v sshd Mon Aug 8 04:40:00 2022 root 344 32616 0 04:39 ? 00:00:00 [ssh] root 428 32616 0 04:39 ? 00:00:00 [ssh] root 516 32616 0 04:39 ? 00:00:00 [ssh] root 601 32616 0 04:39 ? 00:00:00 [ssh] root 32622 32616 0 04:38 ? 00:00:00 [ssh] root 32692 32616 0 04:38 ? 00:00:00 [ssh] root 32740 32616 0 04:39 ? 00:00:00 [ssh]

How can we avoid it?

thanks.

Im having a little look at adding sonic to the list of supported platforms so I can use Boxen to stand up Sonic relatively quickly on MacOs (and anywhere else). I really like the minimalist/streamlined code here and have largely scaffolded from the PanOS material because it looks so similar. No Go expert and welcome stewarding of my ability wherever its needed!

-Fixes a possible bug where LoadConfig always overrides current config. Now user can merge with the existing config instead of overriding.

-Fixes junos shell and root-shell regex to adapt to the wider range of junos devices.

Examples os shell prompts: start shell:

• %
• [vrf:foo] regress@EVOvFOOBAR_RE0-re0:~$

start shell user root:

• root@vMX1_RE:/var/home/regress #
• [vrf:foo] root@EVOvFOOBAR_RE0-re0:~#

Tested these regexes here: https://regex101.com/

-Also added a LoadConfig example as it didn't exist in any of the examples.

What do you think if the examples run against the DevNet always-on IOS-XE device, so anyone can run the examples?

I also added them to the Make file, so you can execute them from the root (all but the NETCONF one)

$ make
base-example                   Run Base example
lint                           Run linters
net-channellog-example         Run Network Channel Log example
net-factory-example            Run Network Factory example
net-interactive-example        Run Network Interactive example
net-log-example                Run Network logging example
net-onopen-example             Run Network On Open example
net-simple-example             Run Network simple example
test                           Test execution

I added a note in the readme on how you would run the examples.

$  make net-simple-example 
found prompt: 
csr1000v-1#


sent command 'show version', output received:
 Cisco IOS XE Software, Version 16.09.03
Cisco IOS Software [Fuji], Virtual XE Software (X86_64_LINUX_IOSD-UNIVERSALK9-M), Version 16.9.3, RELEASE SOFTWARE (fc2)
Technical Support: http://www.cisco.com/techsupport
Copyright (c) 1986-2019 by Cisco Systems, Inc.
Compiled Wed 20-Mar-19 07:56 by mcpre
...

PS: I added a bit more space between the outputs to make it more clear.

Hello,

While testing get-operationnal queries on a juniper device that uses netconf 1.0, I was getting timeouts... so I looked around in the debugger and found out that the forceSelfClosingTags (which needs to be used with juniper because of a bug), erases the query alltogether.

See the following MWP (I copied the forceSelfClosingTags function and hard-coded the pattern):

package main

import (
	"fmt"
	"regexp"
)

func main() {
	req := `
	<?xml version="1.0" encoding="UTF-8"?><rpc xmlns="urn:ietf:params:xml:ns:netconf:base:1.0" message-id="101"><get><filter type="subtree">
	<routing-policy xmlns="http://openconfig.net/yang/routing-policy"></routing-policy>
	</filter></get></rpc>]]>]]>
	`
	print("filter")
	print(req)
	print("self-close")
	print(string(forceSelfClosingTags([]byte(req))))
}

func forceSelfClosingTags(b []byte) []byte {

	emptyTagIdxs := regexp.MustCompile(`<(\w+)></\w+>`).FindAllSubmatchIndex(b, -1)

	var nb []byte

	for _, idx := range emptyTagIdxs {
		// get everything in b up till the first of the submatch indexes (this is the start of an
		// "empty" <thing></thing> tag), then get the name of the tag and put it in a self-closing
		// tag.
		nb = append(b[0:idx[0]], fmt.Sprintf("<%s/>", b[idx[2]:idx[3]])...) //nolint: gocritic

		// finally, append everything *after* the submatch indexes
		nb = append(nb, b[len(b)-(len(b)-idx[1]):]...)
	}

	return nb
}

and it's output:

filter
        <?xml version="1.0" encoding="UTF-8"?><rpc xmlns="urn:ietf:params:xml:ns:netconf:base:1.0" message-id="101"><get><filter type="subtree">
        <routing-policy xmlns="http://openconfig.net/yang/routing-policy"></routing-policy>
        </filter></get></rpc>]]>]]>
        self-close                                                                                                                                                                                          

It seems to me that the pattern is too restrictive. A self-closing tag may contain any valid attributes.

I've experimented with this replace function instead (with some success):

func forceSelfClosingTags(b []byte) []byte {

	cleanStr := regexp.MustCompile(`<([^>/]+?)>\s*</[\w-]+>`).ReplaceAllString(string(b), "<$1/>")

	return []byte(cleanStr)
}