Merge pull request #12 from lanl/develop

Modernize the code base

Author: dmopalmer

.vscode/launch.json

@@ -0,0 +1,16 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Python: Current File",
+            "type": "python",
+            "request": "launch",
+            "program": "${file}",
+            "console": "integratedTerminal",
+            "justMyCode": false
+        }
+    ]
+}

CHANGES.md

@@ -0,0 +1,17 @@
+# Changes for swiftbat_python
+
+## v 0.1.4 2023-08-05
+
+- Started CHANGES.md file
+- Replace `pyephem` with `skyfield`
+  - pyephem is no longer supported
+- Pointing history replaced screen-scraping with `swifttool` database access
+  - Appeared as DOS on the scraped website when I ran too often
+- General decrufting
+  - reformatted with black
+  - Code still suffers from being the first Python I ever wrote, on python2.6
+  - Some (but not much) type hinting)
+- Requirements updated:
+  - Python >= 3.9
+  - astropy >= 5
+  - skyfield >= 1.4

setup.py

@@ -1,6 +1,6 @@
 from setuptools import setup
 
-copyright="""
+copyright = """
 Copyright (c) 2018, Triad National Security, LLC. All rights reserved.
  
 This program was produced under U.S. Government contract
@@ -28,27 +28,38 @@
 """
 
 try:
-    with open("README.md", 'r') as f:
+    with open("README.md", "r") as f:
         long_description = f.read()
 except FileNotFoundError:
-    long_description = ''
+    long_description = ""
 
 setup(
-    name='swiftbat',
-    version='0.1.2a3',
-    packages=['swiftbat'],
-    package_data={'':['catalog', 'recent_bcttb.fits.gz']},
-    url='https://github.com/lanl/swiftbat_python/',
-    license='BSD-3-Clause',
-    author='David M. Palmer',
-    author_email='palmer@lanl.gov',
-    description='Routines for dealing with data from BAT on the Neil Gehrels Swift Observatory',
+    name="swiftbat",
+    version="0.1.4",
+    packages=["swiftbat"],
+    package_data={"": ["catalog", "recent_bcttb.fits.gz"]},
+    url="https://github.com/lanl/swiftbat_python/",
+    license="BSD-3-Clause",
+    author="David M. Palmer",
+    author_email="palmer@lanl.gov",
+    description="Routines for dealing with data from BAT on the Neil Gehrels Swift Observatory",
     long_description=long_description,
-    long_description_content_type='text/markdown',
-    entry_points={'console_scripts': ['swinfo=swiftbat.swinfo:main']},
-    install_requires = ['beautifulsoup4', 'pyephem', 'astropy', 'astroquery', 'numpy'],
-    classifiers=['Development Status :: 3 - Alpha', 'Intended Audience :: Science/Research', 'Programming Language :: Python',
-        'Programming Language :: Python :: 3',
-        'Topic :: Scientific/Engineering :: Astronomy', ],
-    python_requires='>=3.6',
+    long_description_content_type="text/markdown",
+    entry_points={"console_scripts": ["swinfo=swiftbat.swinfo:swinfo_main"]},
+    install_requires=[
+        "astropy>=5",
+        "astroquery",
+        "numpy",
+        "python-dateutil",
+        "skyfield>=1.4",
+        "swifttools",
+    ],
+    classifiers=[
+        "Development Status :: 4 - Beta",
+        "Intended Audience :: Science/Research",
+        "Programming Language :: Python",
+        "Programming Language :: Python :: 3",
+        "Topic :: Scientific/Engineering :: Astronomy",
+    ],
+    python_requires=">=3.9",
 )

swiftbat/__init__.py

@@ -17,9 +17,9 @@
 
 """
 
-from __future__ import print_function, division, absolute_import
-
+from .sfmisc import sftime, sfts, loadsfephem
 from .clockinfo import utcf
 from .swutil import *
 from .swinfo import *
 from .batcatalog import BATCatalog
+from . import generaldir

swiftbat/batcatalog.py

@@ -25,12 +25,12 @@
 from astropy.io import fits
 import numpy as np
 from pathlib import Path
-from swiftbat import simbadnames, simbadlocation
+from swiftbat import simbadlocation
 import re
 
 
-class BATCatalog():
-    filebasename = 'recent_bcttb.fits.gz'
+class BATCatalog:
+    filebasename = "recent_bcttb.fits.gz"
     thisdir = Path(__file__).parent
 
     def __init__(self, catalogfilename=None):
@@ -86,34 +86,36 @@ def getall(self, item):
 
     def simbadmatch(self, item, tolerance=0.2):
         """
-        What rows match the catalogued 
-        :param item: 
-        :param tolerance: 
-        :return: 
+        What rows match the catalogued
+        :param item:
+        :param tolerance:
+        :return:
         """
         ra, dec = simbadlocation(item)
         return self.positionmatch(ra, dec, tolerance)
 
     def positionmatch(self, radeg, decdeg, tolerance=0.2):
         rascale = np.cos(np.deg2rad(decdeg))
-        tol2 = tolerance ** 2
-        raoff = (((self.cattable['RA_OBJ'] - radeg) + 180) % 360 - 180)  # Handle the 360-0 wrap
-        dist2 = (raoff * rascale) ** 2 + (self.cattable['DEC_OBJ'] - decdeg) ** 2
+        tol2 = tolerance**2
+        raoff = (
+            (self.cattable["RA_OBJ"] - radeg) + 180
+        ) % 360 - 180  # Handle the 360-0 wrap
+        dist2 = (raoff * rascale) ** 2 + (self.cattable["DEC_OBJ"] - decdeg) ** 2
         return self.cattable[dist2 < tol2]
 
     def allnames(self):
-        return set([row['NAME'] for row in self.cattable if row['NAME']])
+        return set([row["NAME"] for row in self.cattable if row["NAME"]])
 
     def makeindices(self):
         self.bycatnum = {}
         self.byname = {}
         self.bysimplename = {}
         for row in self.cattable:
-            if not row['CATNUM']:
+            if not row["CATNUM"]:
                 continue
-            self.bycatnum.setdefault(row['CATNUM'], []).append(row)
-            self.byname.setdefault(row['NAME'], []).append(row)
-            self.bysimplename.setdefault(self.simplename(row['NAME']), []).append(row)
+            self.bycatnum.setdefault(row["CATNUM"], []).append(row)
+            self.byname.setdefault(row["NAME"], []).append(row)
+            self.bysimplename.setdefault(self.simplename(row["NAME"]), []).append(row)
 
     @lru_cache(maxsize=0)
     def _cattable(self, catalogfilename=None):
@@ -132,4 +134,4 @@ def simplename(self, name):
         :param name:
         :return:
         """
-        return re.sub('[^a-z0-9]', '', name.lower())
+        return re.sub("[^a-z0-9]", "", name.lower())

swiftbat/clockinfo.py

@@ -1,11 +1,11 @@
-from __future__ import division, print_function
-
 import os
+import sys
 import glob
 import datetime
-import re
 from pathlib import Path
-import ftplib
+from .generaldir import httpDir
+from astropy.io import fits
+
 
 """
 From the FITS file:
@@ -50,36 +50,38 @@
 
 
 class clockErrData:
-    # URL is never used on David Palmer's machine (updates handled by ...swift-trend/getit)
-    # clockurl = "https://heasarc.gsfc.nasa.gov/FTP/swift/calib_data/sc/bcf/clock/"
-    clockurl = "ftps://heasarc.gsfc.nasa.gov/caldb/data/swift/mis/bcf/clock/"
-    clockhost = 'heasarc.gsfc.nasa.gov'
-    clockhostdir = '/caldb/data/swift/mis/bcf/clock/'
-    clockfile_regex = 'swclockcor20041120v\d*.fits'
+    clockurl = "https://heasarc.gsfc.nasa.gov/FTP/swift/calib_data/sc/bcf/clock/"
+    clockhost = "heasarc.gsfc.nasa.gov"
+    clockhostdir = "/caldb/data/swift/mis/bcf/clock/"
+    clockfile_regex = "swclockcor20041120v\d*.fits"
+    clockfilepattern = "swclockcor20041120v*.fits"
     # FIXME this should be derived from the dotswift params
-    clocklocalsearchpath = ['/opt/data/Swift/swift-trend/clock',
-                            os.path.expanduser('~/.swift/swiftclock'),
-                            '/tmp/swiftclock']
-    clockfilepattern = 'swclockcor20041120v*.fits'
+    clocklocalsearchpath = [
+        "/opt/data/Swift/swift-trend/clock",
+        os.path.expanduser("~/.swift/swiftclock"),
+        "/tmp/swiftclock",
+    ]
 
     def __init__(self):
         try:
             self._clockfile = self.clockfile()
-            from astropy.io import fits
+
             f = fits.open(self._clockfile)
             # copies are needed to prevent pyfits from holding open a file handle for each item
-            self._tstart = f[1].data.field('TSTART').copy()
-            self._tstop = f[1].data.field('TSTOP').copy()
-            self._toffset = f[1].data.field('TOFFSET').copy()
-            self._c0 = f[1].data.field('C0').copy()
-            self._c1 = f[1].data.field('C1').copy()
-            self._c2 = f[1].data.field('C2').copy()
+            self._tstart = f[1].data.field("TSTART").copy()
+            self._tstop = f[1].data.field("TSTOP").copy()
+            self._toffset = f[1].data.field("TOFFSET").copy()
+            self._c0 = f[1].data.field("C0").copy()
+            self._c1 = f[1].data.field("C1").copy()
+            self._c2 = f[1].data.field("C2").copy()
             f.close()
         except:
             self._clockfile = ""
             raise RuntimeError("No clock UTCF file")
 
-    def utcf(self, t, trow=None):  # Returns value in seconds to be added to MET to give correct UTCF
+    def utcf(
+        self, t, trow=None
+    ):  # Returns value in seconds to be added to MET to give correct UTCF
         if not self._clockfile:
             return 0.0, "No UTCF file"
         if trow is None:
@@ -93,9 +95,13 @@ def utcf(self, t, trow=None):  # Returns value in seconds to be added to MET to
         else:
             row = row[-1]
             if self._tstop[row] + caveat_time < t:
-                caveats += "Time %.1f days after clock correction interval\n" % ((t - self._tstop[row]) / 86400)
+                caveats += "Time %.1f days after clock correction interval\n" % (
+                    (t - self._tstop[row]) / 86400
+                )
         ddays = (t - self._tstart[row]) / 86400.0
-        tcorr = self._toffset[row] + 1e-6 * (self._c0[row] + ddays * (self._c1[row] + ddays * self._c2[row]))
+        tcorr = self._toffset[row] + 1e-6 * (
+            self._c0[row] + ddays * (self._c1[row] + ddays * self._c2[row])
+        )
         return -tcorr, caveats
 
     def updateclockfiles(self, clockdir, ifolderthan_days=30, test_days=1):
@@ -108,39 +114,44 @@ def updateclockfiles(self, clockdir, ifolderthan_days=30, test_days=1):
         """
         testfile = os.path.join(clockdir, "clocktest")
         try:
-            clockfile = sorted(list(glob.glob(os.path.join(clockdir, self.clockfilepattern))))[-1]
-            age = datetime.datetime.utcnow() - datetime.datetime.fromtimestamp(os.path.getmtime(clockfile))
+            clockfile = sorted(
+                list(glob.glob(os.path.join(clockdir, self.clockfilepattern)))
+            )[-1]
+            age = datetime.datetime.utcnow() - datetime.datetime.fromtimestamp(
+                os.path.getmtime(clockfile)
+            )
             if age.total_seconds() < (86400 * ifolderthan_days):
                 return
             # Check no more than once a day
-            testage = datetime.datetime.utcnow() - datetime.datetime.fromtimestamp(os.path.getmtime(testfile))
+            testage = datetime.datetime.utcnow() - datetime.datetime.fromtimestamp(
+                os.path.getmtime(testfile)
+            )
             if testage.total_seconds() < (86400 * test_days):
                 return
         except:
             pass
-        # Requires wget.  If this is a problem, use ftplib.FTP
-        # os.system(
-        #     "wget -q --directory-prefix=%s --no-host --no-clobber --cut-dirs=6 -r %s"
-        #     % (clockdir, self.clockurl) )
         try:
-            ftps = ftplib.FTP_TLS(self.clockhost)
-            ftps.login()    # anonymous
-            ftps.prot_p()   # for ftps
-            ftps.cwd(self.clockhostdir)
-            clockreg = re.compile(self.clockfile_regex)
-            ftplatest = sorted([f for f in ftps.nlst() if clockreg.match(f)])[-1]
-            locallatest = Path(clockdir).joinpath(ftplatest)
-            if not locallatest.exists():
-                with open(locallatest, "wb") as newfile:
-                    ftps.retrbinary(f'RETR {ftplatest}', newfile.write)
-            open(testfile,'w').write(' ') # touch
+            clockremotedir = httpDir(self.clockurl)
+            clockremote = sorted(clockremotedir.getMatches("", self.clockfile_regex))[
+                -1
+            ]
+            locallatest = Path(clockdir).joinpath(Path(clockremote).name)
+            clockremotedir.copyToFile(clockremote, locallatest)
+            open(testfile, "w").write(" ")  # touch
         except Exception as e:
             print(e, file=sys.stdout)
 
-
     def clockfile(self):
         for clockdir in self.clocklocalsearchpath:
+            # Directory must exist and have readable clock files in it.
             if os.path.exists(clockdir):
+                try:
+                    clockfile = sorted(
+                        list(glob.glob(os.path.join(clockdir, self.clockfilepattern)))
+                    )[-1]
+                    clockdata = fits.getdata(clockfile)
+                except:
+                    continue
                 break
         else:
             for clockdir in self.clocklocalsearchpath:
@@ -154,11 +165,15 @@ def clockfile(self):
                         pass
             else:
                 try:
-                    os.makedirs(clockdir)   # Force directory to exist in the temp directory
+                    os.makedirs(
+                        clockdir
+                    )  # Force directory to exist in the temp directory
                 except FileExistsError:
                     pass
         self.updateclockfiles(clockdir)
-        clockfile = sorted(list(glob.glob(os.path.join(clockdir, self.clockfilepattern))))[-1]
+        clockfile = sorted(
+            list(glob.glob(os.path.join(clockdir, self.clockfilepattern)))
+        )[-1]
         return clockfile
 
 
@@ -175,9 +190,7 @@ def utcf(met, printCaveats=True, returnCaveats=False):
         theClockData = clockErrData()
     try:
         uc = [theClockData.utcf(t_) for t_ in met]
-        # http://muffinresearch.co.uk/archives/2007/10/16/python-transposing-lists-with-map-and-zip/
-        # Not valid after Python 2.7
-        u, c = map(None, *uc)
+        u, c = zip(*uc)
         if printCaveats and any(c):
             print("\n".join(["**** " + c_ for c_ in c if c_]))
     except TypeError: