[pull] master from sqlmapproject:master

data/txt/sha256sums.txt

@@ -171,7 +171,7 @@ b13462712ec5ac07541dba98631ddcda279d210b838f363d15ac97a1413b67a2  lib/core/bigar
 a6397b10de7ae7c56ed6b0fa3b3c58eb7a9dbede61bf93d786e73258175c981e  lib/core/compat.py
 a9997e97ebe88e0bf7efcf21e878bc5f62c72348e5aba18f64d6861390a4dcf2  lib/core/convert.py
 c03dc585f89642cfd81b087ac2723e3e1bb3bfa8c60e6f5fe58ef3b0113ebfe6  lib/core/data.py
-ca06a0e9d66a58e74ef994d53f9b3cd2ebaed98735bbab99854054235a8083d6  lib/core/datatype.py
+6acb645b1f285b21673c70824b03f6209acc5993b50e50da5ed2c713a30626f5  lib/core/datatype.py
 70fb2528e580b22564899595b0dff6b1bc257c6a99d2022ce3996a3d04e68e4e  lib/core/decorators.py
 147823c37596bd6a56d677697781f34b8d1d1671d5a2518fbc9468d623c6d07d  lib/core/defaults.py
 6b366f897e66b9df39df2ee45fef77d46efb7a2d4e294440d3aa7dc1b2f4cedf  lib/core/dicts.py
@@ -182,13 +182,13 @@ a033f92d136c707a25927c2383125ddb004d4283db62c004dcd67c3fc242bb1c  lib/core/dump.
 914a13ee21fd610a6153a37cbe50830fcbd1324c7ebc1e7fc206d5e598b0f7ad  lib/core/log.py
 02a2264324caa249154e024a01bcd7cc40dbca4d647d5d10a50654b4415a6d77  lib/core/optiondict.py
 c1cb56f2a43e9f2f6b25d5f3d504e856ea21df6fc14af5e37b1000feef2bdb5a  lib/core/option.py
-8171f6ee33e7742f06bb3014a28324496374beddee7b378ace10a26414a97762  lib/core/patch.py
+9a213f91c8ad468466bd92e5e5805040f904055eb607fb2ed75b4c0e30b8accd  lib/core/patch.py
 49c0fa7e3814dfda610d665ee02b12df299b28bc0b6773815b4395514ddf8dec  lib/core/profiling.py
 03db48f02c3d07a047ddb8fe33a757b6238867352d8ddda2a83e4fec09a98d04  lib/core/readlineng.py
 48797d6c34dd9bb8a53f7f3794c85f4288d82a9a1d6be7fcf317d388cb20d4b3  lib/core/replication.py
 0b8c38a01bb01f843d94a6c5f2075ee47520d0c4aa799cecea9c3e2c5a4a23a6  lib/core/revision.py
 888daba83fd4a34e9503fe21f01fef4cc730e5cde871b1d40e15d4cbc847d56c  lib/core/session.py
-1418691b5449412e60c693b6afc2f12b00051c1e280d2261762a36f094e0da66  lib/core/settings.py
+8508162b2a95e54102ee8aec95888d7e2061d73b7d0e9ecd47d4f5e22ca94820  lib/core/settings.py
 cd5a66deee8963ba8e7e9af3dd36eb5e8127d4d68698811c29e789655f507f82  lib/core/shell.py
 bcb5d8090d5e3e0ef2a586ba09ba80eef0c6d51feb0f611ed25299fbb254f725  lib/core/subprocessng.py
 d35650179816193164a5f177102f18379dfbe6bb6d40fbb67b78d907b41c8038  lib/core/target.py
@@ -206,7 +206,7 @@ c5b258be7485089fac9d9cd179960e774fbd85e62836dc67cce76cc028bb6aeb  lib/parse/hand
 1ad9054cd8476a520d4e2c141085ae45d94519df5c66f25fac41fe7d552ab952  lib/parse/html.py
 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3  lib/parse/__init__.py
 d2e771cdacef25ee3fdc0e0355b92e7cd1b68f5edc2756ffc19f75d183ba2c73  lib/parse/payloads.py
-80d26a30abe948faf817a14f746cc8b3e2341ea8286830cccaae253b8ac0cdff  lib/parse/sitemap.py
+455ab0ec63e55cd56ce4a884b85bdc089223155008cab0f3696da5a33118f95b  lib/parse/sitemap.py
 1be3da334411657461421b8a26a0f2ff28e1af1e28f1e963c6c92768f9b0847c  lib/request/basicauthhandler.py
 a1c638493ecdc5194db7186bbfed815c6eed2344f2607cac8c9fa50534824266  lib/request/basic.py
 bc61bc944b81a7670884f82231033a6ac703324b34b071c9834886a92e249d0e  lib/request/chunkedhandler.py
@@ -618,7 +618,7 @@ edf23e7105539d700a1ae1bc52436e57e019b345a7d0227e4d85b6353ef535fa  thirdparty/ide
 d846fdc47a11a58da9e463a948200f69265181f3dbc38148bfe4141fade10347  thirdparty/identywaf/LICENSE
 e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855  thirdparty/__init__.py
 879d96f2460bc6c79c0db46b5813080841c7403399292ce76fe1dc0a6ed353d8  thirdparty/keepalive/__init__.py
-f517561115b0cfaa509d0d4216cd91c7de92c6a5a30f1688fdca22e4cd52b8f8  thirdparty/keepalive/keepalive.py
+c7ac7253fa450030f9c42f11bb19689055bb8c39621bcfbeca856ba3c9342760  thirdparty/keepalive/keepalive.py
 e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855  thirdparty/magic/__init__.py
 4d89a52f809c28ce1dc17bb0c00c775475b8ce01c2165942877596a6180a2fd8  thirdparty/magic/magic.py
 e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855  thirdparty/multipart/__init__.py

lib/core/datatype.py

@@ -38,6 +38,8 @@ def __getattr__(self, item):
         Maps values to attributes
         Only called if there *is NOT* an attribute with this name
         """
+        if item.startswith('__') and item.endswith('__'):
+             raise AttributeError(item)
 
         try:
             return self.__getitem__(item)

lib/core/patch.py

@@ -160,6 +160,23 @@ def _releaseLock():
 
         logging._releaseLock = _releaseLock
 
+    from xml.etree import ElementTree as et
+    if not getattr(et, "_patched", False):
+        _real_parse = et.parse
+
+        def _safe_parse(source, parser=None):
+            if parser is None:
+                parser = et.XMLParser()
+                if hasattr(parser, "parser"):
+                    def reject(*args): raise ValueError("XML entities are forbidden")
+                    parser.parser.EntityDeclHandler = reject
+                    parser.parser.UnparsedEntityDeclHandler = reject
+
+            return _real_parse(source, parser=parser)
+
+        et.parse = _safe_parse
+        et._patched = True
+
 def resolveCrossReferences():
     """
     Place for cross-reference resolution

lib/core/settings.py

@@ -20,7 +20,7 @@
 from thirdparty import six
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.10.1.85"
+VERSION = "1.10.2.3"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)

lib/parse/sitemap.py

@@ -17,7 +17,7 @@
 
 abortedFlag = None
 
-def parseSitemap(url, retVal=None):
+def parseSitemap(url, retVal=None, visited=None):
     global abortedFlag
 
     if retVal is not None:
@@ -27,25 +27,41 @@ def parseSitemap(url, retVal=None):
         if retVal is None:
             abortedFlag = False
             retVal = OrderedSet()
+            visited = set()
+
+        if url in visited:
+            return retVal
+
+        visited.add(url)
 
         try:
             content = Request.getPage(url=url, raise404=True)[0] if not abortedFlag else ""
         except _http_client.InvalidURL:
             errMsg = "invalid URL given for sitemap ('%s')" % url
             raise SqlmapSyntaxException(errMsg)
 
-        for match in re.finditer(r"<loc>\s*([^<]+)", content or ""):
-            if abortedFlag:
-                break
-            url = match.group(1).strip()
-            if url.endswith(".xml") and "sitemap" in url.lower():
-                if kb.followSitemapRecursion is None:
-                    message = "sitemap recursion detected. Do you want to follow? [y/N] "
-                    kb.followSitemapRecursion = readInput(message, default='N', boolean=True)
-                if kb.followSitemapRecursion:
-                    parseSitemap(url, retVal)
-            else:
-                retVal.add(url)
+        if content:
+            content = re.sub(r"", "", content, flags=re.DOTALL)
+
+            for match in re.finditer(r"<\w*?loc[^>]*>\s*([^<]+)", content, re.I):
+                if abortedFlag:
+                    break
+
+                foundUrl = match.group(1).strip()
+
+                # Basic validation to avoid junk
+                if not foundUrl.startswith("http"):
+                    continue
+
+                if foundUrl.endswith(".xml") and "sitemap" in foundUrl.lower():
+                    if kb.followSitemapRecursion is None:
+                        message = "sitemap recursion detected. Do you want to follow? [y/N] "
+                        kb.followSitemapRecursion = readInput(message, default='N', boolean=True)
+
+                    if kb.followSitemapRecursion:
+                        parseSitemap(foundUrl, retVal, visited)
+                else:
+                    retVal.add(foundUrl)
 
     except KeyboardInterrupt:
         abortedFlag = True

thirdparty/keepalive/keepalive.py

@@ -12,9 +12,9 @@
 #   Lesser General Public License for more details.
 #
 #   You should have received a copy of the GNU Lesser General Public
-#   License along with this library; if not, write to the 
-#      Free Software Foundation, Inc., 
-#      59 Temple Place, Suite 330, 
+#   License along with this library; if not, write to the
+#      Free Software Foundation, Inc.,
+#      59 Temple Place, Suite 330,
 #      Boston, MA  02111-1307  USA
 
 # This file was part of urlgrabber, a high-level cross-protocol url-grabber
@@ -28,7 +28,7 @@
 >>> keepalive_handler = HTTPHandler()
 >>> opener = _urllib.request.build_opener(keepalive_handler)
 >>> _urllib.request.install_opener(opener)
->>> 
+>>>
 >>> fo = _urllib.request.urlopen('http://www.python.org')
 
 If a connection to a given host is requested, and all of the existing
@@ -154,14 +154,18 @@ def remove(self, connection):
             else:
                 del self._connmap[connection]
                 del self._readymap[connection]
-                self._hostmap[host].remove(connection)
+                try:
+                    self._hostmap[host].remove(connection)
+                except ValueError:
+                    pass
                 if not self._hostmap[host]: del self._hostmap[host]
         finally:
             self._lock.release()
 
     def set_ready(self, connection, ready):
-        try: self._readymap[connection] = ready
-        except KeyError: pass
+        self._lock.acquire()
+        if connection in self._readymap: self._readymap[connection] = ready
+        self._lock.release()
 
     def get_ready_conn(self, host):
         conn = None
@@ -178,10 +182,14 @@ def get_ready_conn(self, host):
         return conn
 
     def get_all(self, host=None):
-        if host:
-            return list(self._hostmap.get(host, []))
-        else:
-            return dict(self._hostmap)
+        self._lock.acquire()
+        try:
+            if host:
+                return list(self._hostmap.get(host, []))
+            else:
+                return dict(self._hostmap)
+        finally:
+            self._lock.release()
 
 class KeepAliveHandler:
     def __init__(self):
@@ -242,9 +250,9 @@ def do_open(self, req):
                 h = self._get_connection(host)
                 if DEBUG: DEBUG.info("creating new connection to %s (%d)",
                                      host, id(h))
-                self._cm.add(host, h, 0)
                 self._start_transaction(h, req)
                 r = h.getresponse()
+                self._cm.add(host, h, 0)
         except (socket.error, _http_client.HTTPException) as err:
             raise _urllib.error.URLError(err)
 
@@ -254,20 +262,20 @@ def do_open(self, req):
         if r.will_close:
             if DEBUG: DEBUG.info('server will close connection, discarding')
             self._cm.remove(h)
+            h.close()
 
         r._handler = self
         r._host = host
         r._url = req.get_full_url()
         r._connection = h
         r.code = r.status
         r.headers = r.msg
-        r.msg = r.reason
 
         if r.status == 200 or not HANDLE_ERRORS:
             return r
         else:
             return self.parent.error('http', req, r,
-                                     r.status, r.msg, r.headers)
+                                     r.status, r.reason, r.headers)
 
     def _reuse_connection(self, h, req, host):
         """start the transaction with a re-used connection
@@ -283,7 +291,7 @@ def _reuse_connection(self, h, req, host):
             # worked.  We'll check the version below, too.
         except (socket.error, _http_client.HTTPException):
             r = None
-        except:
+        except Exception:
             # adding this block just in case we've missed
             # something we will still raise the exception, but
             # lets try and close the connection and remove it
@@ -314,16 +322,16 @@ def _reuse_connection(self, h, req, host):
 
     def _start_transaction(self, h, req):
         try:
-            if req.data:
+            if req.data is not None:
                 data = req.data
                 if hasattr(req, 'selector'):
                     h.putrequest(req.get_method() or 'POST', req.selector, skip_host=req.has_header("Host"), skip_accept_encoding=req.has_header("Accept-encoding"))
                 else:
                     h.putrequest(req.get_method() or 'POST', req.get_selector(), skip_host=req.has_header("Host"), skip_accept_encoding=req.has_header("Accept-encoding"))
-                if 'Content-type' not in req.headers:
+                if not req.has_header('Content-type'):
                     h.putheader('Content-type',
                                 'application/x-www-form-urlencoded')
-                if 'Content-length' not in req.headers:
+                if not req.has_header('Content-length'):
                     h.putheader('Content-length', '%d' % len(data))
             else:
                 if hasattr(req, 'selector'):
@@ -333,20 +341,20 @@ def _start_transaction(self, h, req):
         except (socket.error, _http_client.HTTPException) as err:
             raise _urllib.error.URLError(err)
 
-        if 'Connection' not in req.headers:
-            req.headers['Connection'] = 'keep-alive'
+        if not req.has_header('Connection'):
+            h.putheader('Connection', 'keep-alive')
 
         for args in self.parent.addheaders:
-            if args[0] not in req.headers:
+            if not req.has_header(args[0]):
                 h.putheader(*args)
         for k, v in req.headers.items():
             h.putheader(k, v)
         h.endheaders()
-        if req.data:
+        if req.data is not None:
             h.send(data)
 
     def _get_connection(self, host):
-        return NotImplementedError
+        raise NotImplementedError()
 
 class HTTPHandler(KeepAliveHandler, _urllib.request.HTTPHandler):
     def __init__(self):
@@ -373,8 +381,10 @@ def https_open(self, req):
         return self.do_open(req)
 
     def _get_connection(self, host):
-        try: return self._ssl_factory.get_https_connection(host)
-        except AttributeError: return HTTPSConnection(host)
+        if self._ssl_factory:
+            return self._ssl_factory.get_https_connection(host)
+        else:
+            return HTTPSConnection(host)
 
 class HTTPResponse(_http_client.HTTPResponse):
     # we need to subclass HTTPResponse in order to
@@ -397,9 +407,9 @@ class HTTPResponse(_http_client.HTTPResponse):
 
 
     def __init__(self, sock, debuglevel=0, strict=0, method=None):
-        if method: # the httplib in python 2.3 uses the method arg
-            _http_client.HTTPResponse.__init__(self, sock, debuglevel, method)
-        else: # 2.2 doesn't
+        if method:
+            _http_client.HTTPResponse.__init__(self, sock, debuglevel, method=method)
+        else:
             _http_client.HTTPResponse.__init__(self, sock, debuglevel)
         self.fileno = sock.fileno
         self.code = None
@@ -453,11 +463,11 @@ def read(self, amt=None):
 
     def readline(self, limit=-1):
         data = b""
-        i = self._rbuf.find('\n')
+        i = self._rbuf.find(b'\n')
         while i < 0 and not (0 < limit <= len(self._rbuf)):
             new = self._raw_read(self._rbufsize)
             if not new: break
-            i = new.find('\n')
+            i = new.find(b'\n')
             if i >= 0: i = i + len(self._rbuf)
             self._rbuf = self._rbuf + new
         if i < 0: i = len(self._rbuf)
@@ -468,15 +478,15 @@ def readline(self, limit=-1):
 
     def readlines(self, sizehint = 0):
         total = 0
-        list = []
+        lines = []
         while 1:
             line = self.readline()
             if not line: break
-            list.append(line)
+            lines.append(line)
             total += len(line)
             if sizehint and total >= sizehint:
                 break
-        return list
+        return lines
 
 
 class HTTPConnection(_http_client.HTTPConnection):
@@ -540,10 +550,10 @@ def continuity(url):
     print(format % ('keepalive read', m.hexdigest()))
 
     fo = _urllib.request.urlopen(url)
-    foo = ''
+    foo = b''
     while 1:
         f = fo.readline()
-        if f: foo = foo + f
+        if f: foo += f
         else: break
     fo.close()
     m = md5(foo)