Curl[Banner Grabbing]

curl -si <URL> |grep HTTP/1.1

for i in $(<robots.txt); do echo -n $i; curl -si 10.11.1.50$i |grep HTTP/1.1 |sed 's/HTTP\/1.1/ /g'; done

OR

python3 dirsearch.py -u 10.11.1.50 -w /oscp/bethany/10.11.1.50/loot/robots.txt -e html,php

[Find web page title and links]{.underline}

curl <url> -s -L |grep "title\|href" |sed -e 's/^[[:space:<two closed square brackets>*//'