CsvStore.js 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627
  1. if (!dojo._hasResource["dojox.data.CsvStore"]) { // _hasResource checks added
  2. // by build. Do not use
  3. // _hasResource directly in
  4. // your code.
  5. dojo._hasResource["dojox.data.CsvStore"] = true;
  6. dojo.provide("dojox.data.CsvStore");
  7. dojo.require("dojo.data.util.filter");
  8. dojo.require("dojo.data.util.simpleFetch");
  9. dojo.declare("dojox.data.CsvStore", null, {
  10. // summary:
  11. // The CsvStore implements the dojo.data.api.Read API and reads
  12. // data from files in CSV (Comma Separated Values) format.
  13. // All values are simple string values. References to other items
  14. // are not supported as attribute values in this datastore.
  15. //
  16. // Example data file:
  17. // name, color, age, tagline
  18. // Kermit, green, 12, "Hi, I'm Kermit the Frog."
  19. // Fozzie Bear, orange, 10, "Wakka Wakka Wakka!"
  20. // Miss Piggy, pink, 11, "Kermie!"
  21. //
  22. // Note that values containing a comma must be enclosed with quotes ("")
  23. // Also note that values containing quotes must be escaped with two
  24. // consecutive quotes (""quoted"")
  25. /*
  26. * examples: var csvStore = new dojox.data.CsvStore({url:"movies.csv");
  27. * var csvStore = new
  28. * dojox.data.CsvStore({url:"http://example.com/movies.csv");
  29. */
  30. constructor : function(/* Object */keywordParameters) {
  31. // summary: initializer
  32. // keywordParameters: {url: String}
  33. // keywordParameters: {data: String}
  34. // keywordParameters: {label: String} The column label for the
  35. // column to use for the label returned by getLabel.
  36. this._attributes = []; // e.g. ["Title", "Year", "Producer"]
  37. this._attributeIndexes = {}; // e.g. {Title: 0, Year: 1,
  38. // Producer: 2}
  39. this._dataArray = []; // e.g. [[<Item0>],[<Item1>],[<Item2>]]
  40. this._arrayOfAllItems = []; // e.g. [{_csvId:0,_csvStore:store},...]
  41. this._loadFinished = false;
  42. if (keywordParameters.url) {
  43. this.url = keywordParameters.url;
  44. }
  45. this._csvData = keywordParameters.data;
  46. if (keywordParameters.label) {
  47. this.label = keywordParameters.label;
  48. } else if (this.label === "") {
  49. this.label = undefined;
  50. }
  51. this._storeProp = "_csvStore"; // Property name for the store
  52. // reference on every item.
  53. this._idProp = "_csvId"; // Property name for the Item Id on
  54. // every item.
  55. this._features = {
  56. 'dojo.data.api.Read' : true,
  57. 'dojo.data.api.Identity' : true
  58. };
  59. this._loadInProgress = false; // Got to track the initial load to
  60. // prevent duelling loads of the
  61. // dataset.
  62. this._queuedFetches = [];
  63. },
  64. url : "", // Declarative hook for setting Csv source url.
  65. label : "", // Declarative hook for setting the label attribute.
  66. _assertIsItem : function(/* item */item) {
  67. // summary:
  68. // This function tests whether the item passed in is indeed an item
  69. // in the store.
  70. // item:
  71. // The item to test for being contained by the store.
  72. if (!this.isItem(item)) {
  73. throw new Error("dojox.data.CsvStore: a function was passed an item argument that was not an item");
  74. }
  75. },
  76. _assertIsAttribute : function(/* item || String */attribute) {
  77. // summary:
  78. // This function tests whether the item passed in is indeed a valid
  79. // 'attribute' like type for the store.
  80. // attribute:
  81. // The attribute to test for being contained by the store.
  82. if (!dojo.isString(attribute)) {
  83. throw new Error("dojox.data.CsvStore: a function was passed an attribute argument that was not an attribute object nor an attribute name string");
  84. }
  85. },
  86. /***********************************************************************
  87. * dojo.data.api.Read API
  88. **********************************************************************/
  89. getValue : function( /* item */item,
  90. /* attribute || attribute-name-string */attribute,
  91. /* value? */defaultValue) {
  92. // summary:
  93. // See dojo.data.api.Read.getValue()
  94. // Note that for the CsvStore, an empty string value is the same as
  95. // no value,
  96. // so the defaultValue would be returned instead of an empty string.
  97. this._assertIsItem(item);
  98. this._assertIsAttribute(attribute);
  99. var itemValue = defaultValue;
  100. if (this.hasAttribute(item, attribute)) {
  101. var itemData = this._dataArray[this.getIdentity(item)];
  102. itemValue = itemData[this._attributeIndexes[attribute]];
  103. }
  104. return itemValue; // String
  105. },
  106. getValues : function(/* item */item,
  107. /* attribute || attribute-name-string */attribute) {
  108. // summary:
  109. // See dojo.data.api.Read.getValues()
  110. // CSV syntax does not support multi-valued attributes, so this is
  111. // just a
  112. // wrapper function for getValue().
  113. var value = this.getValue(item, attribute);
  114. return (value ? [value] : []); // Array
  115. },
  116. getAttributes : function(/* item */item) {
  117. // summary:
  118. // See dojo.data.api.Read.getAttributes()
  119. this._assertIsItem(item);
  120. var attributes = [];
  121. var itemData = this._dataArray[this.getIdentity(item)];
  122. for (var i = 0; i < itemData.length; i++) {
  123. // Check for empty string values. CsvStore treats empty strings
  124. // as no value.
  125. if (itemData[i] != "") {
  126. attributes.push(this._attributes[i]);
  127. }
  128. }
  129. return attributes; // Array
  130. },
  131. hasAttribute : function( /* item */item,
  132. /* attribute || attribute-name-string */attribute) {
  133. // summary:
  134. // See dojo.data.api.Read.hasAttribute()
  135. // The hasAttribute test is true if attribute has an index number
  136. // within the item's array length
  137. // AND if the item has a value for that attribute. Note that for the
  138. // CsvStore, an
  139. // empty string value is the same as no value.
  140. this._assertIsItem(item);
  141. this._assertIsAttribute(attribute);
  142. var attributeIndex = this._attributeIndexes[attribute];
  143. var itemData = this._dataArray[this.getIdentity(item)];
  144. return (typeof attributeIndex != "undefined"
  145. && attributeIndex < itemData.length && itemData[attributeIndex] != ""); // Boolean
  146. },
  147. containsValue : function(/* item */item,
  148. /* attribute || attribute-name-string */attribute,
  149. /* anything */value) {
  150. // summary:
  151. // See dojo.data.api.Read.containsValue()
  152. var regexp = undefined;
  153. if (typeof value === "string") {
  154. regexp = dojo.data.util.filter.patternToRegExp(value, false);
  155. }
  156. return this._containsValue(item, attribute, value, regexp); // boolean.
  157. },
  158. _containsValue : function( /* item */item,
  159. /* attribute || attribute-name-string */attribute,
  160. /* anything */value,
  161. /* RegExp? */regexp) {
  162. // summary:
  163. // Internal function for looking at the values contained by the
  164. // item.
  165. // description:
  166. // Internal function for looking at the values contained by the
  167. // item. This
  168. // function allows for denoting if the comparison should be case
  169. // sensitive for
  170. // strings or not (for handling filtering cases where string case
  171. // should not matter)
  172. //
  173. // item:
  174. // The data item to examine for attribute values.
  175. // attribute:
  176. // The attribute to inspect.
  177. // value:
  178. // The value to match.
  179. // regexp:
  180. // Optional regular expression generated off value if value was of
  181. // string type to handle wildcarding.
  182. // If present and attribute values are string, then it can be used
  183. // for comparison instead of 'value'
  184. var values = this.getValues(item, attribute);
  185. for (var i = 0; i < values.length; ++i) {
  186. var possibleValue = values[i];
  187. if (typeof possibleValue === "string" && regexp) {
  188. return (possibleValue.match(regexp) !== null);
  189. } else {
  190. // Non-string matching.
  191. if (value === possibleValue) {
  192. return true; // Boolean
  193. }
  194. }
  195. }
  196. return false; // Boolean
  197. },
  198. isItem : function(/* anything */something) {
  199. // summary:
  200. // See dojo.data.api.Read.isItem()
  201. if (something && something[this._storeProp] === this) {
  202. var identity = something[this._idProp];
  203. if (identity >= 0 && identity < this._dataArray.length) {
  204. return true; // Boolean
  205. }
  206. }
  207. return false; // Boolean
  208. },
  209. isItemLoaded : function(/* anything */something) {
  210. // summary:
  211. // See dojo.data.api.Read.isItemLoaded()
  212. // The CsvStore always loads all items, so if it's an item, then
  213. // it's loaded.
  214. return this.isItem(something); // Boolean
  215. },
  216. loadItem : function(/* item */item) {
  217. // summary:
  218. // See dojo.data.api.Read.loadItem()
  219. // description:
  220. // The CsvStore always loads all items, so if it's an item, then
  221. // it's loaded.
  222. // From the dojo.data.api.Read.loadItem docs:
  223. // If a call to isItemLoaded() returns true before loadItem() is
  224. // even called,
  225. // then loadItem() need not do any work at all and will not even
  226. // invoke
  227. // the callback handlers.
  228. },
  229. getFeatures : function() {
  230. // summary:
  231. // See dojo.data.api.Read.getFeatures()
  232. return this._features; // Object
  233. },
  234. getLabel : function(/* item */item) {
  235. // summary:
  236. // See dojo.data.api.Read.getLabel()
  237. if (this.label && this.isItem(item)) {
  238. return this.getValue(item, this.label); // String
  239. }
  240. return undefined; // undefined
  241. },
  242. getLabelAttributes : function(/* item */item) {
  243. // summary:
  244. // See dojo.data.api.Read.getLabelAttributes()
  245. if (this.label) {
  246. return [this.label]; // array
  247. }
  248. return null; // null
  249. },
  250. // The dojo.data.api.Read.fetch() function is implemented as
  251. // a mixin from dojo.data.util.simpleFetch.
  252. // That mixin requires us to define _fetchItems().
  253. _fetchItems : function( /* Object */keywordArgs,
  254. /* Function */findCallback,
  255. /* Function */errorCallback) {
  256. // summary:
  257. // See dojo.data.util.simpleFetch.fetch()
  258. var self = this;
  259. var filter = function(requestArgs, arrayOfAllItems) {
  260. var items = null;
  261. if (requestArgs.query) {
  262. items = [];
  263. var ignoreCase = requestArgs.queryOptions
  264. ? requestArgs.queryOptions.ignoreCase
  265. : false;
  266. // See if there are any string values that can be regexp
  267. // parsed first to avoid multiple regexp gens on the
  268. // same value for each item examined. Much more efficient.
  269. var regexpList = {};
  270. for (var key in requestArgs.query) {
  271. var value = requestArgs.query[key];
  272. if (typeof value === "string") {
  273. regexpList[key] = dojo.data.util.filter
  274. .patternToRegExp(value, ignoreCase);
  275. }
  276. }
  277. for (var i = 0; i < arrayOfAllItems.length; ++i) {
  278. var match = true;
  279. var candidateItem = arrayOfAllItems[i];
  280. for (var key in requestArgs.query) {
  281. var value = requestArgs.query[key];
  282. if (!self._containsValue(candidateItem, key, value,
  283. regexpList[key])) {
  284. match = false;
  285. }
  286. }
  287. if (match) {
  288. items.push(candidateItem);
  289. }
  290. }
  291. } else {
  292. // We want a copy to pass back in case the parent wishes to
  293. // sort the array. We shouldn't allow resort
  294. // of the internal list so that multiple callers can get
  295. // lists and sort without affecting each other.
  296. if (arrayOfAllItems.length > 0) {
  297. items = arrayOfAllItems
  298. .slice(0, arrayOfAllItems.length);
  299. }
  300. }
  301. findCallback(items, requestArgs);
  302. };
  303. if (this._loadFinished) {
  304. filter(keywordArgs, this._arrayOfAllItems);
  305. } else {
  306. if (this.url !== "") {
  307. // If fetches come in before the loading has finished, but
  308. // while
  309. // a load is in progress, we have to defer the fetching to
  310. // be
  311. // invoked in the callback.
  312. if (this._loadInProgress) {
  313. this._queuedFetches.push({
  314. args : keywordArgs,
  315. filter : filter
  316. });
  317. } else {
  318. this._loadInProgress = true;
  319. var getArgs = {
  320. url : self.url,
  321. handleAs : "text"
  322. };
  323. var getHandler = dojo.xhrGet(getArgs);
  324. getHandler.addCallback(function(data) {
  325. self._processData(data);
  326. filter(keywordArgs, self._arrayOfAllItems);
  327. self._handleQueuedFetches();
  328. });
  329. getHandler.addErrback(function(error) {
  330. self._loadInProgress = false;
  331. throw error;
  332. });
  333. }
  334. } else if (this._csvData) {
  335. this._processData(this._csvData);
  336. this._csvData = null;
  337. filter(keywordArgs, this._arrayOfAllItems);
  338. } else {
  339. throw new Error("dojox.data.CsvStore: No CSV source data was provided as either URL or String data input.");
  340. }
  341. }
  342. },
  343. close : function(
  344. /* dojo.data.api.Request || keywordArgs || null */request) {
  345. // summary:
  346. // See dojo.data.api.Read.close()
  347. },
  348. // -------------------------------------------------------------------
  349. // Private methods
  350. _getArrayOfArraysFromCsvFileContents : function(
  351. /* string */ csvFileContents) {
  352. /*
  353. * summary: Parses a string of CSV records into a nested array
  354. * structure. description: Given a string containing CSV records,
  355. * this method parses the string and returns a data structure
  356. * containing the parsed content. The data structure we return is an
  357. * array of length R, where R is the number of rows (lines) in the
  358. * CSV data. The return array contains one sub-array for each CSV
  359. * line, and each sub-array contains C string values, where C is the
  360. * number of columns in the CSV data.
  361. */
  362. /*
  363. * example: For example, given this CSV string as input: "Title,
  364. * Year, Producer \n Alien, 1979, Ridley Scott \n Blade Runner,
  365. * 1982, Ridley Scott" this._dataArray will be set to: [["Alien",
  366. * "1979", "Ridley Scott"], ["Blade Runner", "1982", "Ridley
  367. * Scott"]] And this._attributes will be set to: ["Title", "Year",
  368. * "Producer"] And this._attributeIndexes will be set to: {
  369. * "Title":0, "Year":1, "Producer":2 }
  370. */
  371. if (dojo.isString(csvFileContents)) {
  372. var lineEndingCharacters = new RegExp("\r\n|\n|\r");
  373. var leadingWhiteSpaceCharacters = new RegExp("^\\s+", 'g');
  374. var trailingWhiteSpaceCharacters = new RegExp("\\s+$", 'g');
  375. var doubleQuotes = new RegExp('""', 'g');
  376. var arrayOfOutputRecords = [];
  377. var arrayOfInputLines = csvFileContents
  378. .split(lineEndingCharacters);
  379. for (var i = 0; i < arrayOfInputLines.length; ++i) {
  380. var singleLine = arrayOfInputLines[i];
  381. if (singleLine.length > 0) {
  382. var listOfFields = singleLine.split(',');
  383. var j = 0;
  384. while (j < listOfFields.length) {
  385. var space_field_space = listOfFields[j];
  386. var field_space = space_field_space.replace(
  387. leadingWhiteSpaceCharacters, ''); // trim
  388. // leading
  389. // whitespace
  390. var field = field_space.replace(
  391. trailingWhiteSpaceCharacters, ''); // trim
  392. // trailing
  393. // whitespace
  394. var firstChar = field.charAt(0);
  395. var lastChar = field.charAt(field.length - 1);
  396. var secondToLastChar = field.charAt(field.length
  397. - 2);
  398. var thirdToLastChar = field
  399. .charAt(field.length - 3);
  400. if (field.length === 2 && field == "\"\"") {
  401. listOfFields[j] = ""; // Special case empty
  402. // string field.
  403. } else if ((firstChar == '"')
  404. && ((lastChar != '"') || ((lastChar == '"')
  405. && (secondToLastChar == '"') && (thirdToLastChar != '"')))) {
  406. if (j + 1 === listOfFields.length) {
  407. // alert("The last field in record " + i + "
  408. // is corrupted:\n" + field);
  409. return null; // null
  410. }
  411. var nextField = listOfFields[j + 1];
  412. listOfFields[j] = field_space + ',' + nextField;
  413. listOfFields.splice(j + 1, 1); // delete
  414. // element [j+1]
  415. // from the list
  416. } else {
  417. if ((firstChar == '"') && (lastChar == '"')) {
  418. field = field.slice(1, (field.length - 1)); // trim
  419. // the
  420. // "
  421. // characters
  422. // off
  423. // the
  424. // ends
  425. field = field.replace(doubleQuotes, '"'); // replace
  426. // ""
  427. // with
  428. // "
  429. }
  430. listOfFields[j] = field;
  431. j += 1;
  432. }
  433. }
  434. arrayOfOutputRecords.push(listOfFields);
  435. }
  436. }
  437. // The first item of the array must be the header row with
  438. // attribute names.
  439. this._attributes = arrayOfOutputRecords.shift();
  440. for (var i = 0; i < this._attributes.length; i++) {
  441. // Store the index of each attribute
  442. this._attributeIndexes[this._attributes[i]] = i;
  443. }
  444. this._dataArray = arrayOfOutputRecords; // Array
  445. }
  446. },
  447. _processData : function(/* String */data) {
  448. this._getArrayOfArraysFromCsvFileContents(data);
  449. this._arrayOfAllItems = [];
  450. for (var i = 0; i < this._dataArray.length; i++) {
  451. this._arrayOfAllItems.push(this._createItemFromIdentity(i));
  452. }
  453. this._loadFinished = true;
  454. this._loadInProgress = false;
  455. },
  456. _createItemFromIdentity : function(/* String */identity) {
  457. var item = {};
  458. item[this._storeProp] = this;
  459. item[this._idProp] = identity;
  460. return item; // Object
  461. },
  462. /***********************************************************************
  463. * dojo.data.api.Identity API
  464. **********************************************************************/
  465. getIdentity : function(/* item */item) {
  466. // summary:
  467. // See dojo.data.api.Identity.getIdentity()
  468. if (this.isItem(item)) {
  469. return item[this._idProp]; // String
  470. }
  471. return null; // null
  472. },
  473. fetchItemByIdentity : function(/* Object */keywordArgs) {
  474. // summary:
  475. // See dojo.data.api.Identity.fetchItemByIdentity()
  476. // Hasn't loaded yet, we have to trigger the load.
  477. if (!this._loadFinished) {
  478. var self = this;
  479. if (this.url !== "") {
  480. // If fetches come in before the loading has finished, but
  481. // while
  482. // a load is in progress, we have to defer the fetching to
  483. // be
  484. // invoked in the callback.
  485. if (this._loadInProgress) {
  486. this._queuedFetches.push({
  487. args : keywordArgs
  488. });
  489. } else {
  490. this._loadInProgress = true;
  491. var getArgs = {
  492. url : self.url,
  493. handleAs : "text"
  494. };
  495. var getHandler = dojo.xhrGet(getArgs);
  496. getHandler.addCallback(function(data) {
  497. var scope = keywordArgs.scope
  498. ? keywordArgs.scope
  499. : dojo.global;
  500. try {
  501. self._processData(data);
  502. var item = self
  503. ._createItemFromIdentity(keywordArgs.identity);
  504. if (!self.isItem(item)) {
  505. item = null;
  506. }
  507. if (keywordArgs.onItem) {
  508. keywordArgs.onItem.call(scope, item);
  509. }
  510. self._handleQueuedFetches();
  511. } catch (error) {
  512. if (keywordArgs.onError) {
  513. keywordArgs.onError.call(scope, error);
  514. }
  515. }
  516. });
  517. getHandler.addErrback(function(error) {
  518. this._loadInProgress = false;
  519. if (keywordArgs.onError) {
  520. var scope = keywordArgs.scope
  521. ? keywordArgs.scope
  522. : dojo.global;
  523. keywordArgs.onError.call(scope, error);
  524. }
  525. });
  526. }
  527. } else if (this._csvData) {
  528. self._processData(self._csvData);
  529. self._csvData = null;
  530. var item = self
  531. ._createItemFromIdentity(keywordArgs.identity);
  532. if (!self.isItem(item)) {
  533. item = null;
  534. }
  535. if (keywordArgs.onItem) {
  536. var scope = keywordArgs.scope
  537. ? keywordArgs.scope
  538. : dojo.global;
  539. keywordArgs.onItem.call(scope, item);
  540. }
  541. }
  542. } else {
  543. // Already loaded. We can just look it up and call back.
  544. var item = this._createItemFromIdentity(keywordArgs.identity);
  545. if (!this.isItem(item)) {
  546. item = null;
  547. }
  548. if (keywordArgs.onItem) {
  549. var scope = keywordArgs.scope
  550. ? keywordArgs.scope
  551. : dojo.global;
  552. keywordArgs.onItem.call(scope, item);
  553. }
  554. }
  555. },
  556. getIdentityAttributes : function(/* item */item) {
  557. // summary:
  558. // See dojo.data.api.Identity.getIdentifierAttributes()
  559. // Identity isn't a public attribute in the item, it's the row
  560. // position index.
  561. // So, return null.
  562. return null;
  563. },
  564. _handleQueuedFetches : function() {
  565. // summary:
  566. // Internal function to execute delayed request in the store.
  567. // Execute any deferred fetches now.
  568. if (this._queuedFetches.length > 0) {
  569. for (var i = 0; i < this._queuedFetches.length; i++) {
  570. var fData = this._queuedFetches[i];
  571. var delayedFilter = fData.filter;
  572. var delayedQuery = fData.args;
  573. if (delayedFilter) {
  574. delayedFilter(delayedQuery, this._arrayOfAllItems);
  575. } else {
  576. this.fetchItemByIdentity(fData.args);
  577. }
  578. }
  579. this._queuedFetches = [];
  580. }
  581. }
  582. });
  583. // Mix in the simple fetch implementation to this class.
  584. dojo.extend(dojox.data.CsvStore, dojo.data.util.simpleFetch);
  585. }