1 /***************************************************************************
3 * Project ___| | | | _ \| |
5 * | (__| |_| | _ <| |___
6 * \___|\___/|_| \_\_____|
8 * Copyright (C) 1998 - 2011, Daniel Stenberg, <daniel@haxx.se>, et al.
10 * This software is licensed as described in the file COPYING, which
11 * you should have received as part of this distribution. The terms
12 * are also available at http://curl.haxx.se/docs/copyright.html.
14 * You may opt to use, copy, modify, merge, publish, distribute and/or sell
15 * copies of the Software, and permit persons to whom the Software is
16 * furnished to do so, under the terms of the COPYING file.
18 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
19 * KIND, either express or implied.
21 ***************************************************************************/
22 /* Download a document and use libtidy to parse the HTML.
23 * Written by Jeff Pohlmeyer
25 * LibTidy => http://tidy.sourceforge.net
27 * gcc -Wall -I/usr/local/include tidycurl.c -lcurl -ltidy -o tidycurl
32 #include <tidy/tidy.h>
33 #include <tidy/buffio.h>
34 #include <curl/curl.h>
36 /* curl write callback, to fill tidy's input buffer... */
37 uint write_cb(char *in, uint size, uint nmemb, TidyBuffer *out)
41 tidyBufAppend( out, in, r );
45 /* Traverse the document tree */
46 void dumpNode(TidyDoc doc, TidyNode tnod, int indent )
49 for ( child = tidyGetChild(tnod); child; child = tidyGetNext(child) )
51 ctmbstr name = tidyNodeGetName( child );
54 /* if it has a name, then it's an HTML tag ... */
56 printf( "%*.*s%s ", indent, indent, "<", name);
57 /* walk the attribute list */
58 for ( attr=tidyAttrFirst(child); attr; attr=tidyAttrNext(attr) ) {
59 printf(tidyAttrName(attr));
60 tidyAttrValue(attr)?printf("=\"%s\" ",
61 tidyAttrValue(attr)):printf(" ");
66 /* if it doesn't have a name, then it's probably text, cdata, etc... */
69 tidyNodeGetText(doc, child, &buf);
70 printf("%*.*s\n", indent, indent, buf.bp?(char *)buf.bp:"");
73 dumpNode( doc, child, indent + 4 ); /* recursive */
78 int main(int argc, char **argv )
81 char curl_errbuf[CURL_ERROR_SIZE];
83 TidyBuffer docbuf = {0};
84 TidyBuffer tidy_errbuf = {0};
87 curl = curl_easy_init();
88 curl_easy_setopt(curl, CURLOPT_URL, argv[1]);
89 curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, curl_errbuf);
90 curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L);
91 curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
92 curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_cb);
95 tidyOptSetBool(tdoc, TidyForceOutput, yes); /* try harder */
96 tidyOptSetInt(tdoc, TidyWrapLen, 4096);
97 tidySetErrorBuffer( tdoc, &tidy_errbuf );
100 curl_easy_setopt(curl, CURLOPT_WRITEDATA, &docbuf);
101 err=curl_easy_perform(curl);
103 err = tidyParseBuffer(tdoc, &docbuf); /* parse the input */
105 err = tidyCleanAndRepair(tdoc); /* fix any problems */
107 err = tidyRunDiagnostics(tdoc); /* load tidy error buffer */
109 dumpNode( tdoc, tidyGetRoot(tdoc), 0 ); /* walk the tree */
110 fprintf(stderr, "%s\n", tidy_errbuf.bp); /* show errors */
116 fprintf(stderr, "%s\n", curl_errbuf);
119 curl_easy_cleanup(curl);
120 tidyBufFree(&docbuf);
121 tidyBufFree(&tidy_errbuf);
127 printf( "usage: %s <url>\n", argv[0] );