# Required python moduldes for opening FITS files and plotting
import numpy as np
import os.path as op
import glob
import matplotlib.pyplot as plt
from matplotlib import gridspec
from astropy.io import fits
from astropy.table import Table, hstack
import astropy.units as u
from astropy.coordinates import SkyCoord
from astropy.wcs import WCS
from astropy.visualization import make_lupton_rgb, ZScaleInterval
%matplotlib inline
# fill in the path to the HETDEX Source Catalog 1 files
path_to_sc1 = "/home/jovyan/Hobby-Eberly-Public/HETDEX/catalogs/hetdex_source_catalog_1"
version = 'v3.2' # Change to latest version.
Two catalogs make up HETDEX Source Catalog 1:
1. The Source Observation Table: hetdex_sc1_vX.dat/.fits/.ecsv
With SPECTRA arrays included: hetdex_sc1_spec_vX.fits
One row per source observation. The table provides basic coordinates/redshift/source information for each observation of a unique astornomical source. The larger file hetdex_sc1_spec_vX.fits contains the same info from the first table plus addition data units of spectral array data.
2. The Detection Information Table: hetdex_sc1_detinfo_vX.fits
One row per line or continuum detection. Bright sources can be comprised of multiple line or continuum emission. This catalog provides specific detection information such as line parameter info (S/N, line flux, line width), observational data and instrument details.Multiple formats are provided for the source table. As suggested by Astropy v5.1 developers, we provide the recommended .ecsv format which contains data masking, description and quantity info. We also provide a simple .dat ascii file.
source_table = Table.read(op.join( path_to_sc1, 'hetdex_sc1_{}.ecsv'.format(version) ) )
# Column Info
source_table.info
<Table length=232650>
name dtype unit description n_bad
------------- ------- ------------------- -------------------------------------------------------------------------- ------
source_name str26 HETDEX IAU designation 0
source_id int64 HETDEX Source Identifier 0
shotid int64 integer represent observation ID: int( date+obsid) 0
RA float32 deg source_id right ascension (ICRS deg) 0
DEC float32 deg source_id declination (ICRS deg) 0
gmag float32 sdss-g magnitude measured in HETDEX spectrum 0
Av float32 applied dust correction in V band 0
z_hetdex float32 HETDEX spectroscopic redshift 0
z_hetdex_src str8 HETDEX spectroscopic redshift source 0
z_hetdex_conf float32 0 to 1 confidence HETDEX spectroscopic redshift source 0
source_type str4 options are 'star', 'lae', 'agn', 'lzg', 'oii', 'none' 0
n_members int64 number of detections in the source group 0
detectid int64 emission line or detection ID 0
field str10 field ID: cosmos, goods-n, dex-fall, dex-spring 0
flux_aper float32 1e-17 erg / (cm2 s) Dust corrected, OII line flux measured in elliptical galaxy aperture 115942
flux_aper_err float32 1e-17 erg / (cm2 s) error in flux_aper 121558
flag_aper int64 1 = OII aperture line flux used, -1= PSF-line flux used from "flux" column 0
major float32 arcsec Major axis of aperture ellipse of resolved OII galaxy defined by imaging 108572
minor float32 arcsec Minor axis of aperture ellipse of resolved OII galaxy defined by imaging 108572
theta float32 angle in aperture ellipse 108572
lum_lya float64 erg / s Lya luminosity in ergs/s 179976
lum_lya_err float64 erg / s error in lum_lya 179976
lum_oii float64 erg / s best OII line luminosity 105242
lum_oii_err float64 erg / s error in lum_oii 110840
flux_lya float32 1e-17 erg / (cm2 s) dust corrected Lya line flux) in ergs/s 179976
flux_lya_err float32 1e-17 erg / (cm2 s) error in flux_lya 179976
flux_oii float32 1e-17 erg / (cm2 s) best OII line flux 105241
flux_oii_err float32 1e-17 erg / (cm2 s) error in flux_oii 110839
sn float32 signal-to-noise for line emission 52486
apcor float32 aperture correction applied to spectrum at 4500\AA 0
If spectral data is desired, a fits file contains spectral array data matching each row of the Source Observation Table
hdu = fits.open( op.join( path_to_sc1, 'hetdex_sc1_spec_{}.fits'.format(version)))
# The source table can also be accessed by astropy Table class or through HDU1
#source_table = Table.read( op.join( path_to_sc1, 'hetdex_sc1_spec_{}.fits'.format(version)))
hdu.info()
Filename: /home/jovyan/Hobby-Eberly-Public/HETDEX/catalogs/hetdex_source_catalog_1/hetdex_sc1_spec_v3.2.fits No. Name Ver Type Cards Dimensions Format 0 PRIMARY 1 PrimaryHDU 4 () 1 INFO 1 BinTableHDU 88 232650R x 30C [26A, K, K, E, E, E, E, E, 8A, E, 4A, K, K, 12A, E, E, K, E, E, E, D, D, D, D, E, E, E, E, E, E] 2 SPEC 1 ImageHDU 10 (1036, 232650) float32 3 SPEC_ERR 1 ImageHDU 9 (1036, 232650) float32 4 SPEC_OBS 1 ImageHDU 10 (1036, 232650) float32 5 SPEC_OBS_ERR 1 ImageHDU 9 (1036, 232650) float32 6 APCOR 1 ImageHDU 8 (1036, 232650) float32 7 WAVELENGTH 1 ImageHDU 8 (1036,) float64
spec = hdu['SPEC'].data
spec_err = hdu['SPEC_ERR'].data
wave_rect = hdu['WAVELENGTH'].data
#spec unit is:
u.Unit( hdu['SPEC'].header['BUNIT'])
# wavelength unit is:
u.Unit( hdu['WAVELENGTH'].header['BUNIT'])
sel_lae = source_table['source_type'] == 'lae'
print('There are {} LAES in the catalog'.format(np.sum(sel_lae)))
There are 52681 LAES in the catalog
source_ids_lae = source_table['source_id'][sel_lae]
# Plot of all LAE spectra in redshift order as an example
lae_spec_table = hstack( [source_table[sel_lae], spec[sel_lae], spec_err[sel_lae]])
lae_spec_table.rename_column('col0_2', 'spec')
lae_spec_table.rename_column('col0_3', 'spec_err')
lae_spec_table.sort('z_hetdex')
plt.figure(figsize=(4, 10))
plt.imshow( lae_spec_table['spec'], aspect=0.05, vmin=-0.01, vmax=2)
plt.title('HETDEX LAE spectra in increasing redshift order')
plt.xlabel('wavelength/z_hetdex')
plt.ylabel('HETDEX 1D LAE spectra')
Text(0, 0.5, 'HETDEX 1D LAE spectra')
# Create array of coordinates for all HETDEX source members
source_coords = SkyCoord(ra = source_table['RA'], dec= source_table['DEC'])
coord = SkyCoord(ra=220.21432*u.deg, dec=52.095898*u.deg)
sel_match = source_coords.separation(coord) < 1.*u.arcsec
redshift = source_table['z_hetdex'][sel_match][0]
lya_wave = 1216* (redshift + 1)
source_id = source_table['source_id'][sel_match][0]
name = source_table['source_name'][sel_match][0]
plt.plot( wave_rect, spec[sel_match][0])
plt.xlim(lya_wave-50, lya_wave+50)
plt.ylabel(r'spec (10$^-17$ ergs/s/cm$^2$/$\AA$)')
plt.xlabel(r'wavelength ($\AA$)')
plt.title('{} source_id={}'.format(name, source_id))
Text(0.5, 1.0, 'HETDEX J144051.44+520545.2 source_id=2140100250906')
det_table = Table.read(op.join(path_to_sc1, 'hetdex_sc1_detinfo_{}.ecsv'.format(version)))
# Column Info
det_table.info
<Table length=297261>
name dtype unit description n_bad
------------------------ ------- ---------------------------- ------------------------------------------------------------------------------- ------
source_id int64 HETDEX Source Identifier 0
source_name str26 HETDEX IAU designation 0
RA float32 deg source_id right ascension (ICRS deg) 0
DEC float32 deg source_id declination (ICRS deg) 0
z_hetdex float32 HETDEX spectroscopic redshift 0
z_hetdex_src str8 HETDEX spectroscopic redshift source 0
z_hetdex_conf float32 0 to 1 confidence HETDEX spectroscopic redshift source 0
source_type str4 options are 'star', 'lae', 'agn', 'lzg', 'oii', 'none' 0
detectid int64 emission line or detection ID 0
selected_det bool best detectid for Lya flux or OII line flux 0
det_type str4 detection type: 'line' or 'continuum' 0
line_id str4 line identification at observed wavelength (wave) assuming redshift of z_hetdex 0
RA_det float32 deg detectid right ascension (ICRS deg) 0
DEC_det float32 deg detectid declination (ICRS deg) 0
src_separation float32 deg distance between detectid (RA_det, DEC_det) and the source center (RA, DEC) 0
n_members int64 number of detections in the source group 0
gmag_err float32 mcmc uncertainty in gmag 2082
gmag float32 sdss-g magnitude measured in HETDEX spectrum 0
Av float32 applied dust correction in V band 0
ebv float32 applied selective extinction 0
wave float32 Angstrom central wavelength of line emission 0
wave_err float32 Angstrom mcmc error in wave 0
flux float32 1e-17 erg / (cm2 s) dust corrected line flux at "wave" 60907
flux_err float32 1e-17 erg / (cm2 s) mcmc error in dust corrected line flux 60907
flux_obs float32 1e-17 erg / (cm2 s) observed line flux 60907
flux_obs_err float32 1e-17 erg / (cm2 s) mcmc error in observed line flux 60907
flux_aper float32 1e-17 erg / (cm2 s) Dust corrected, OII line flux measured in elliptical galaxy aperture 180472
flux_aper_err float32 1e-17 erg / (cm2 s) error in flux_aper 186101
flux_aper_obs float32 1e-17 erg / (cm2 s) OII line flux measured in elliptical aperture 172625
flux_aper_obs_err float32 1e-17 erg / (cm2 s) error in flag_aper_obs 178879
flag_aper int64 1 = OII aperture line flux used, -1= PSF-line flux used from "flux" column 0
sigma float32 Angstrom sigma linewidth in gaussian line fit (\AA) 0
sigma_err float32 Angstrom mcmc error in sigma linewidth (\AA) 0
continuum float32 1e-17 erg / (Angstrom cm2 s) local fitted observed continuum 60907
continuum_err float32 1e-17 erg / (Angstrom cm2 s) mcmc error in continuum 60907
continuum_obs float32 1e-17 erg / (Angstrom cm2 s) local fitted observed continuum 60907
continuum_obs_err float32 1e-17 erg / (Angstrom cm2 s) mcmc error in continuum 60907
sn float32 signal-to-noise for line emission 60907
sn_err float32 mcmc error in signal-to-noise 0
chi2 float32 reduced $\chi^2$ quality of line fit 0
chi2_err float32 mcmc uncertainty in reduced $\chi^2$ 0
flux_noise_1sigma_obs float32 1e-17 erg / (cm2 s) observed 1 sigma flux sensitivity 60907
flux_noise_1sigma float32 1e-17 erg / (cm2 s) dust corrected 1 sigma flux sensitivity 60907
apcor float32 aperture correction applied to spectrum at 4500\AA 0
counterpart_mag float32 selected closest counterpart mag from imaging data 13172
counterpart_mag_err float32 uncertainty in counterpart_mag 13172
counterpart_dist float32 distance to closest counterpart 13172
counterpart_catalog_name str16 image catalog source of counterpart 0
counterpart_filter_name str5 image filter of countpart 0
plya_classification float32 0 to 1 on likelihood line is Lya 60907
best_z float32 ELiXeR best redshift 61015
best_pz float32 confidence in best_z 61015
z_diagnose float32 Best fit redshift from Diagnose 59
cls_diagnose str7 Diagnose classfication."STAR", "GALAXY", "QSO", "UNKNOWN" 0
stellartype str1 diagnose spectral type classification for stars 0
agn_flag float32 -1 not an AGN, 0 broad lien soruce, 1 confident AGN 0
wave_group_id int64 id for 3D clustering at common ra, dec, wave 0
wave_group_a float32 deg semi-major axis from 3D FOF clustering 247536
wave_group_b float32 deg semi-minor axis from 3D FOF clustering 247536
wave_group_pa float32 positional angle from 3D FOF clustering 247536
wave_group_ra float32 mean ra from 3D FOF clustering 247536
wave_group_dec float32 mean dec from 3D FOF clustering 247536
wave_group_wave float32 mean wavelength from 3D FOF clustering 247536
fwhm float32 measured seeing of the observation in arcsec 0
throughput float32 relative spectral response at 4540 assuming a 360 s nominal exposure 0
shotid int64 integer represent observation ID: int( date+obsid) 0
field str10 field ID: cosmos, goods-n, dex-fall, dex-spring 0
date int32 date 0
obsid int32 observation number 0
multiframe str20 string identifier for the ifuslot/specid/ifuid/amp combination 0
fiber_id str38 string identifier for the highest weight fiber 0
weight float32 flux weight of the highest weight fiber 0
x_raw int32 x value on the CCD of the detection (ds9 x value) 0
y_raw int32 y value on the CCD of the detection (ds9 y value) 0
x_ifu float32 x position in the ifu in arcsec 0
y_ifu float32 y position in the ifu in arcsec 0
ra_aper float32 Right Ascension of aperture center of imaging counterpart in degrees 172620
dec_aper float32 Declination of aperture center of imaging counterpart in degrees 172620
catalog_name_aper str16 Imaging source for measuring OII resolved apertures 0
filter_name_aper str5 Filter of Imaging used for measuring OII resolved apertures 0
dist_aper float32 distance between aperture center and detectid position in arcsec 172620
mag_aper float32 photometric magnitude in aperture in imaging source 172620
mag_aper_err float32 photometric magnitude error in aperture in imaging source 172620
major float32 arcsec Major axis of aperture ellipse of resolved OII galaxy defined by imaging 172620
minor float32 arcsec Minor axis of aperture ellipse of resolved OII galaxy defined by imaging 172620
theta float32 angle in aperture ellipse 172620
Let's query the detection info table for a relatively nearby, bright galaxy that is composed of serveral line detections to demonstrate the content of the Detection Info Table. Nearby galaxies and bright sources such as AGN can be composed of many detections. This is because line emission at different spatial regions and wavelengths will result in multiple detections in the detection search. The source will also likely have a complementary continuum detetion if it is bright (g < 21)
sel_big_oii = (det_table['source_type'] == 'oii') & (det_table['major'] > 6)
sel_center_ifu = (np.abs( det_table['x_ifu']) < 5) & (np.abs(det_table['y_ifu']) < 5)
selected_det = (det_table['selected_det'] == True) & (det_table['n_members'] >6)
sel = sel_big_oii & sel_center_ifu & selected_det
print(' There are {} matches '.format(np.sum(sel)))
There are 79 matches
# Pick random object in list and plot up the source_id
index = np.where(sel)[0][0]
sid = det_table['source_id'][index]
coords = SkyCoord(ra=det_table['RA'][index]*u.deg, dec=det_table['DEC'][index]*u.deg)
# Get Imaging data from Legacy Survey API
fits_file = 'https://www.legacysurvey.org/viewer/fits-cutout?ra={}&dec={}&layer=ls-dr9&width=80&height=80&pixscale=0.25&bands=grz'.format(coords.ra.deg, coords.dec.deg)
hdu_ls = fits.open(fits_file)
wcs_ls = WCS( hdu_ls[0].header).dropaxis(2)
WARNING: FITSFixedWarning: 'cdfix' made the change 'Success'. [astropy.wcs.wcs]
redshift = det_table['z_hetdex'][index]
source_type = det_table['source_type'][index]
grp = det_table[ det_table['source_id'] == sid]
grp.sort('gmag')
# Get Spectrum from source_table
hdu = fits.open(op.join(path_to_sc1, 'hetdex_sc1_spec_{}.fits'.format(version)))
source_table = hdu['INFO'].data
spec = hdu['SPEC'].data
sel_source = source_table['source_id'] == sid
spectra = spec[sel_source][0]
plt.figure(figsize=(13,3))
gs = gridspec.GridSpec(1, 2, width_ratios=[1,3])
ax1 = plt.subplot(gs[0], projection=wcs_ls)
im_zscale = ZScaleInterval(contrast=0.5, krej=1.1)
im_vmin, im_vmax = im_zscale.get_limits(values=hdu_ls[0].data[2])
plt.imshow(hdu_ls[0].data[2], origin='lower', cmap=plt.get_cmap('gray_r'), vmin=im_vmin, vmax=im_vmax )
sel_line = (grp["det_type"] == "line")
if np.sum(sel_line) >= 1:
plt.scatter(
grp["RA_det"][sel_line],
grp["DEC_det"][sel_line],
transform=ax1.get_transform("world"),
marker="x",
color="orange",
linewidth=2,
s=50,
# zorder=100,
label="line emission",
)
sel_cont = grp["det_type"] == "cont"
if np.sum(sel_cont) >= 1:
plt.scatter(
grp["RA_det"][sel_cont],
grp["DEC_det"][sel_cont],
transform=ax1.get_transform("world"),
marker="x",
color="green",
linewidth=2,
s=50,
label="continuum",
)
lon = ax1.coords[0]
lat = ax1.coords[1]
lon.set_axislabel('RA', minpad=0.5)
lat.set_axislabel('Dec', minpad=-0.6)
lon.set_ticklabel(exclude_overlapping=True)
ax2 = plt.subplot(gs[1])
plt.plot(wave_rect, spectra, linewidth=1.2, color='tab:blue')#, yerr=spec_table['spec1d_err'])
plt.xlabel(r'$\lambda$ ($\AA$)')
plt.ylabel('f$_\\lambda$ (10$^{-17}$ ergs/s/cm$^2$/$\AA$)')
plt.xlim(3540, 5450)
selw = (wave_rect > 3540) & (wave_rect < 5450)
y2 = np.max(spectra[selw])
y1 = np.min(spectra[selw])
if y1 > 0:
y1 = 0
# plot all emission lines detected in det_table related to the source_id
for line in np.array( np.unique(grp['line_id'])):
if line == 'null':
continue
sel_line = grp['line_id'] == line
plt.bar(grp['wave'][sel_line][0], height=2*y2, width=30, bottom=y1, color='orange', alpha=0.3)
label = '{} [{}]'.format( grp['detectid'][sel_line][0], line)
if np.isfinite( grp['wave'][sel_line][0]):
plt.text(grp['wave'][sel_line][0]-70, 0.1*y2, label, rotation=90, fontsize=10)
plt.axhline(0, color='tab:grey', linestyle='dashed')
plt.text(0.05, 0.7, 'z={:6.4f}'.format(redshift), transform=ax2.transAxes, color='tab:red', fontsize=20)
plt.text(0.05, 0.85, 'source_id={}'.format(sid), transform=ax2.transAxes, fontsize=14, color='black')
plt.ylim(y1,1.1*y2)
plt.subplots_adjust(wspace=0.3, hspace=0)
plt.tight_layout()
