001 /*
002 * Copyright 2011 The Kuali Foundation.
003 *
004 * Licensed under the Educational Community License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 * http://www.opensource.org/licenses/ecl2.php
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 */
016 package org.kuali.kfs.module.cg.service.impl;
017
018 import java.io.BufferedReader;
019 import java.io.IOException;
020 import java.io.InputStream;
021 import java.io.InputStreamReader;
022 import java.util.Calendar;
023 import java.util.Collection;
024 import java.util.Comparator;
025 import java.util.HashMap;
026 import java.util.List;
027 import java.util.Map;
028 import java.util.SortedMap;
029 import java.util.TreeMap;
030
031 import org.apache.commons.lang.StringUtils;
032 import org.apache.commons.net.ftp.FTPClient;
033 import org.apache.commons.net.ftp.FTPReply;
034 import org.kuali.kfs.module.cg.CGConstants;
035 import org.kuali.kfs.module.cg.batch.CfdaBatchStep;
036 import org.kuali.kfs.module.cg.businessobject.CFDA;
037 import org.kuali.kfs.module.cg.businessobject.CfdaUpdateResults;
038 import org.kuali.kfs.module.cg.service.CfdaService;
039 import org.kuali.kfs.sys.KFSPropertyConstants;
040 import org.kuali.kfs.sys.context.SpringContext;
041 import org.kuali.rice.kns.service.BusinessObjectService;
042 import org.kuali.rice.kns.service.DateTimeService;
043 import org.kuali.rice.kns.service.ParameterService;
044
045 import au.com.bytecode.opencsv.CSVReader;
046
047 public class CfdaServiceImpl implements CfdaService {
048 private static org.apache.log4j.Logger LOG = org.apache.log4j.Logger.getLogger(CfdaServiceImpl.class);
049
050 private BusinessObjectService businessObjectService;
051 protected static Comparator cfdaComparator;
052
053 static {
054 cfdaComparator = new Comparator() {
055 public int compare(Object o1, Object o2) {
056 String lhs = (String) o1;
057 String rhs = (String) o2;
058 return lhs.compareTo(rhs);
059 }
060 };
061 }
062
063 /**
064 * @return
065 * @throws IOException
066 */
067 public SortedMap<String, CFDA> getGovCodes() throws IOException {
068 Calendar calendar = SpringContext.getBean(DateTimeService.class).getCurrentCalendar();
069 SortedMap<String, CFDA> govMap = new TreeMap<String, CFDA>();
070
071 // ftp://ftp.cfda.gov/programs09187.csv
072 String govURL = SpringContext.getBean(ParameterService.class).getParameterValue(CfdaBatchStep.class, CGConstants.SOURCE_URL_PARAMETER);
073 String fileName = StringUtils.substringAfterLast(govURL, "/");
074 govURL = StringUtils.substringBeforeLast(govURL, "/");
075 if (StringUtils.contains(govURL, "ftp://")) {
076 govURL = StringUtils.remove(govURL, "ftp://");
077 }
078
079 // need to pull off the '20' in 2009
080 String year = "" + calendar.get(Calendar.YEAR);
081 year = year.substring(2, 4);
082 fileName = fileName + year;
083
084 // the last 3 numbers in the file name are the day of the year, but the files are from "yesterday"
085 fileName = fileName + String.format("%03d", calendar.get(Calendar.DAY_OF_YEAR) - 1);
086 fileName = fileName + ".csv";
087
088 LOG.info("Getting government file: " + fileName + " for update");
089
090 InputStream inputStream = null;
091 FTPClient ftp = new FTPClient();
092 try {
093 ftp.connect(govURL);
094 int reply = ftp.getReplyCode();
095
096 if (!FTPReply.isPositiveCompletion(reply)) {
097 LOG.error("FTP connection to server not established.");
098 throw new IOException("FTP connection to server not established.");
099 }
100
101 boolean loggedIn = ftp.login("anonymous", "");
102 if (!loggedIn) {
103 LOG.error("Could not login as anonymous.");
104 throw new IOException("Could not login as anonymous.");
105 }
106
107 LOG.info("Successfully connected and logged in");
108
109 inputStream = ftp.retrieveFileStream(fileName);
110 if (inputStream != null) {
111 LOG.info("reading input stream");
112 InputStreamReader screenReader = new InputStreamReader(inputStream);
113 BufferedReader screen = new BufferedReader(screenReader);
114
115 CSVReader csvReader = new CSVReader(screenReader, ',', '"', 1);
116 List<String[]> lines = csvReader.readAll();
117 for (String[] line : lines) {
118 String title = line[0];
119 String number = line[1];
120
121 CFDA cfda = new CFDA();
122 cfda.setCfdaNumber(number);
123 cfda.setCfdaProgramTitleName(title);
124
125 govMap.put(number, cfda);
126 }
127 }
128
129 ftp.logout();
130 ftp.disconnect();
131 }
132 finally {
133 if (ftp.isConnected()) {
134 ftp.disconnect();
135 }
136 }
137
138 return govMap;
139 }
140
141 /**
142 * @return
143 * @throws IOException
144 */
145 public SortedMap<String, CFDA> getKfsCodes() throws IOException {
146 Collection allCodes = businessObjectService.findAll(CFDA.class);
147
148 SortedMap<String, CFDA> kfsMapAll = new TreeMap<String, CFDA>(cfdaComparator);
149 for (Object o : allCodes) {
150 CFDA c = (CFDA) o;
151 kfsMapAll.put(c.getCfdaNumber(), c);
152 }
153 return kfsMapAll;
154 }
155
156 /**
157 *
158 */
159 public CfdaUpdateResults update() throws IOException {
160
161 CfdaUpdateResults results = new CfdaUpdateResults();
162 Map<String, CFDA> govMap = null;
163
164 try {
165 govMap = getGovCodes();
166 }
167 catch (IOException ioe) {
168 LOG.error("Error connecting to URL resource: " + ioe.getMessage(), ioe);
169 StringBuilder builder = new StringBuilder();
170 builder.append("No updates took place.\n");
171 builder.append(ioe.getMessage());
172 results.setMessage(builder.toString());
173 return results;
174 }
175 Map<String, CFDA> kfsMap = getKfsCodes();
176
177 results.setNumberOfRecordsInKfsDatabase(kfsMap.keySet().size());
178 results.setNumberOfRecordsRetrievedFromWebSite(govMap.keySet().size());
179
180 for (Object key : kfsMap.keySet()) {
181
182 CFDA cfdaKfs = kfsMap.get(key);
183 CFDA cfdaGov = govMap.get(key);
184
185 if (cfdaKfs.getCfdaMaintenanceTypeId().startsWith("M")) {
186 // Leave it alone. It's maintained manually.
187 results.setNumberOfRecordsNotUpdatedBecauseManual(1 + results.getNumberOfRecordsNotUpdatedBecauseManual());
188 }
189 else if (cfdaKfs.getCfdaMaintenanceTypeId().startsWith("A")) {
190
191 if (cfdaGov == null) {
192 if (cfdaKfs.isActive()) {
193 cfdaKfs.setActive(false);
194 businessObjectService.save(cfdaKfs);
195 results.setNumberOfRecordsDeactivatedBecauseNoLongerOnWebSite(results.getNumberOfRecordsDeactivatedBecauseNoLongerOnWebSite() + 1);
196 }
197 else {
198 // Leave it alone for historical purposes
199 results.setNumberOfRecrodsNotUpdatedForHistoricalPurposes(results.getNumberOfRecrodsNotUpdatedForHistoricalPurposes() + 1);
200 }
201 }
202 else {
203 if (cfdaKfs.isActive()) {
204 results.setNumberOfRecordsUpdatedBecauseAutomatic(results.getNumberOfRecordsUpdatedBecauseAutomatic() + 1);
205 }
206 else {
207 cfdaKfs.setActive(true);
208 results.setNumberOfRecordsReActivated(results.getNumberOfRecordsReActivated() + 1);
209 }
210
211 cfdaKfs.setCfdaProgramTitleName(cfdaGov.getCfdaProgramTitleName());
212 businessObjectService.save(cfdaKfs);
213 }
214 }
215
216 // Remove it from the govMap so we know what codes from the govMap don't already exist in KFS.
217 govMap.remove(key);
218 }
219
220 // What's left in govMap now is just the codes that don't exist in KFS
221 for (String key : govMap.keySet()) {
222 CFDA cfdaGov = govMap.get(key);
223 cfdaGov.setCfdaMaintenanceTypeId("Automatic");
224 cfdaGov.setActive(true);
225 businessObjectService.save(cfdaGov);
226 results.setNumberOfRecordsNewlyAddedFromWebSite(results.getNumberOfRecordsNewlyAddedFromWebSite() + 1);
227 }
228
229 return results;
230 }
231
232 public void setBusinessObjectService(BusinessObjectService businessObjectService) {
233 this.businessObjectService = businessObjectService;
234 }
235
236 public CFDA getByPrimaryId(String cfdaNumber) {
237 if (StringUtils.isBlank(cfdaNumber)) {
238 return null;
239 }
240 return (CFDA) businessObjectService.findByPrimaryKey(CFDA.class, mapPrimaryKeys(cfdaNumber));
241 }
242
243 protected Map<String, Object> mapPrimaryKeys(String cfdaNumber) {
244 Map<String, Object> primaryKeys = new HashMap();
245 primaryKeys.put(KFSPropertyConstants.CFDA_NUMBER, cfdaNumber.trim());
246 return primaryKeys;
247 }
248
249 }