[PATCHv1] Export file in git-remote-mediawiki

[PATCHv1] Export file in git-remote-mediawiki

[Kim Thuat NGUYEN]

From: nguyenkimthuat <nguyenkimthuat@gmail.com>

This patch adds the functionnality to export the file attachements from the local git's repository using the API of mediawiki. It also provides the possibility for
an user to delete a page in the local repository Git which means the page  will be deleted in the wiki site after user do the 'push'.

Signed-off-by: VOLEK Pavel <Pavel.Volek@ensimag.imag.fr>
Signed-off-by: NGUYEN Kim Thuat <Kim-Thuat.Nguyen@ensimag.imag.fr>
Signed-off-by: ROUCHER IGLESIAS Javier <roucherj@ensimag.imag.fr>
Signed-off-by: Matthieu Moy <Matthieu.Moy@imag.fr>
---
 contrib/mw-to-git/git-remote-mediawiki | 126 +++++++++++++++++++++++++--------
 1 file changed, 98 insertions(+), 28 deletions(-)

diff --git a/contrib/mw-to-git/git-remote-mediawiki b/contrib/mw-to-git/git-remote-mediawiki
index c18bfa1..2fd0e5b 100755
--- a/contrib/mw-to-git/git-remote-mediawiki
+++ b/contrib/mw-to-git/git-remote-mediawiki
@@ -275,6 +275,14 @@ sub run_git {
 	return $res;
 }
 
+sub run_git_raw {
+	no encoding 'utf8';
+	open(my $g,"-|","git " . $_[0]); 
+	my $r = do { local $/; <$g> };
+	close($g);
+
+	return $r;
+}
 
 sub get_last_local_revision {
 	# Get note regarding last mediawiki revision
@@ -644,6 +652,11 @@ sub mw_push_file {
 	my $page_deleted = ($new_sha1 eq NULL_SHA1);
 	$complete_file_name = mediawiki_clean_filename($complete_file_name);
 
+	my %hashFiles = get_file_extensions_maybe($complete_file_name);
+	my $path = "File:".$complete_file_name;
+	my @extensions = split(/\./, $complete_file_name);
+	my $extension = pop(@extensions);
+
 	if (substr($complete_file_name,-3) eq ".mw") {
 		my $title = substr($complete_file_name,0,-3);
 
@@ -653,39 +666,74 @@ sub mw_push_file {
 			# special priviledges. A common
 			# convention is to replace the page
 			# with this content instead:
-			$file_content = DELETED_CONTENT;
+			mw_connect_maybe();
+			my $re = $mediawiki->edit( {
+				action => 'delete',
+				title => $title,
+				reason => $summary 
+				} )|| die $mediawiki-> {error}->{code} . ':' . $mediawiki->{error}->{details};
 		} else {
 			$file_content = run_git("cat-file blob $new_sha1");
-		}
-
-		mw_connect_maybe();
 
-		my $result = $mediawiki->edit( {
-			action => 'edit',
-			summary => $summary,
-			title => $title,
-			basetimestamp => $basetimestamps{$newrevid},
-			text => mediawiki_clean($file_content, $page_created),
-				  }, {
-					  skip_encoding => 1 # Helps with names with accentuated characters
-				  });
-		if (!$result) {
-			if ($mediawiki->{error}->{code} == 3) {
-				# edit conflicts, considered as non-fast-forward
-				print STDERR 'Warning: Error ' .
-				    $mediawiki->{error}->{code} .
-				    ' from mediwiki: ' . $mediawiki->{error}->{details} .
-				    ".\n";
-				return ($newrevid, "non-fast-forward");
-			} else {
-				# Other errors. Shouldn't happen => just die()
-				die 'Fatal: Error ' .
-				    $mediawiki->{error}->{code} .
-				    ' from mediwiki: ' . $mediawiki->{error}->{details};
+			mw_connect_maybe();
+	
+			my $result = $mediawiki->edit( {
+				action => 'edit',
+				summary => $summary,
+				title => $title,
+				basetimestamp => $basetimestamps{$newrevid},
+				text => mediawiki_clean($file_content, $page_created),
+					  }, {
+						  skip_encoding => 1 # Helps with names with accentuated characters
+					  });
+			if (!$result) {
+				if ($mediawiki->{error}->{code} == 3) {
+					# edit conflicts, considered as non-fast-forward
+					print STDERR 'Warning: Error ' .
+					    $mediawiki->{error}->{code} .
+				   	 ' from mediwiki: ' . $mediawiki->{error}->{details} .".\n";
+					return ($newrevid, "non-fast-forward");
+				} else {
+					# Other errors. Shouldn't happen => just die()
+					die 'Fatal: Error ' .
+					    $mediawiki->{error}->{code} . ' from mediwiki: ' . $mediawiki->{error}->{details};
+				}
 			}
+			$newrevid = $result->{edit}->{newrevid};
+			print STDERR "Pushed file: $new_sha1 - $title\n";
+		}
+	elsif (exists($hashFiles{$extension}))      
+	{		
+		# Deleting and uploading a file require the priviledge of the user
+		if ($page_deleted) {       
+			mw_connect_maybe();
+			my $res = $mediawiki->edit( {
+				action => 'delete',
+				title => $path,
+				reason => $summary
+				} )|| die $mediawiki-> {error}->{code} . ':' . $mediawiki->{error}->{details};	
+		} else {
+			my $content = run_git_raw("cat-file blob $new_sha1");	
+			if ($content ne "") { 
+				mw_connect_maybe();	
+				$mediawiki->{config}->{upload_url} = "$url/index.php/Special:Upload";
+			
+				my $res = $mediawiki->edit ( {
+					action => 'upload',
+					filename => $complete_file_name,
+					comment => $summary,
+					file => [undef, $complete_file_name, Content => $content ],
+					ignorewarnings=>1,
+					}, {
+						  skip_encoding => 1 # Helps with names with accentuated characters
+						} ) || die $mediawiki-> {error}->{code} . ':' . $mediawiki->{error}->{details};
+				my $last_file_page = $mediawiki->get_page({title =>$path});
+				$newrevid = $last_file_page->{revid};
+				print STDERR "Pushed file: $new_sha1 - $complete_file_name\n";
+			} else {
+				print STDERR "Empty file. Can not upload \n ";
+				}
 		}
-		$newrevid = $result->{edit}->{newrevid};
-		print STDERR "Pushed file: $new_sha1 - $title\n";
 	} else {
 		print STDERR "$complete_file_name not a mediawiki file (Not pushable on this version of git-remote-mediawiki).\n"
 	}
@@ -825,3 +873,25 @@ sub mw_push_revision {
 	print STDOUT "ok $remote\n";
 	return 1;
 }
+
+sub get_file_extensions_maybe {
+	my $file_name = shift;
+	my $est_mw_page = substr($file_name,-3) eq ".mw";
+	if(!$est_mw_page) {
+		mw_connect_maybe();
+
+        	my $query = {
+			action => 'query',
+			meta => 'siteinfo',
+			siprop => 'fileextensions'
+			};
+
+		my $result = $mediawiki->api($query);
+		my @file_extensions= map $_->{ext},@{$result->{query}->{fileextensions}};
+		my %hashFile = map {$_ => 1}@file_extensions;
+
+		return %hashFile;
+	} else {
+		return ;
+	}
+}
-- 
1.7.10.2.552.gaa3bb87

Re: [PATCHv1] Export file in git-remote-mediawiki

[Matthieu Moy]

> Subject: Re: [PATCHv1] Export file in git-remote-mediawiki

We usually write commit subject lines as "subsystem: description", hence

git-remote-mediawiki: export "File:" attachments

Kim Thuat NGUYEN <kim-thuat.nguyen@ensimag.imag.fr> writes:

> From: nguyenkimthuat <nguyenkimthuat@gmail.com>
>
> This patch adds the functionnality to export the file attachements from the local git's repository using the API of mediawiki. It also provides the possibility for
> an user to delete a page in the local repository Git which means the page  will be deleted in the wiki site after user do the 'push'.

Please, avoid long lines (> 80 characters).

> +	open(my $g,"-|","git " . $_[0]); 

Space after , please.

> +	my %hashFiles = get_file_extensions_maybe($complete_file_name);

What does this function do? My first understanding was that it queried
the wiki for allowed file extensions, but why does it need the file
name? It does nothing if $complete_file_name ends with .mw, but then why
do you run it before entering the following if() statement?

>  	if (substr($complete_file_name,-3) eq ".mw") {
>  		my $title = substr($complete_file_name,0,-3);

> @@ -653,39 +666,74 @@ sub mw_push_file {
>  			# special priviledges. A common
>  			# convention is to replace the page
>  			# with this content instead:
> -			$file_content = DELETED_CONTENT;
> +			mw_connect_maybe();
> +			my $re = $mediawiki->edit( {
> +				action => 'delete',
> +				title => $title,
> +				reason => $summary 
> +				} )|| die $mediawiki-> {error}->{code} . ':' . $mediawiki->{error}->{details};

This is an unrelated topic, and should not appear in this patch.

If you want to propagate page deletions, then you also need to deal with
the case where the user is not allowed to do so (very common on
MediaWiki). Also, if you change the code corresponding to the comment
right above, you should update the comment too.

> +	elsif (exists($hashFiles{$extension}))      
> +	{

Brace on the same line as else please.

> +			} else {
> +				print STDERR "Empty file. Can not upload \n ";
> +				}

Broken indentation.

>  	} else {
>  		print STDERR "$complete_file_name not a mediawiki file (Not pushable on this version of git-remote-mediawiki).\n"
>  	}

Isn't the very point of this patch to remove this error message?

> @@ -825,3 +873,25 @@ sub mw_push_revision {
>  	print STDOUT "ok $remote\n";
>  	return 1;
>  }
> +
> +sub get_file_extensions_maybe {
> +	my $file_name = shift;
> +	my $est_mw_page = substr($file_name,-3) eq ".mw";

English please. "est" is french ;-).

-- 
Matthieu Moy
http://www-verimag.imag.fr/~moy/

Re: [PATCHv1] Export file in git-remote-mediawiki

[nguyenki]

On Fri, 08 Jun 2012 16:07:23 +0200, Matthieu Moy wrote:
>> Subject: Re: [PATCHv1] Export file in git-remote-mediawiki
>
> We usually write commit subject lines as "subsystem: description", 
> hence
>
> git-remote-mediawiki: export "File:" attachments
>
Yes, it will be corrected in the next version.

> Kim Thuat NGUYEN <kim-thuat.nguyen@ensimag.imag.fr> writes:
>
>> From: nguyenkimthuat <nguyenkimthuat@gmail.com>
>>
>> This patch adds the functionnality to export the file attachements 
>> from the local git's repository using the API of mediawiki. It also 
>> provides the possibility for
>> an user to delete a page in the local repository Git which means the 
>> page  will be deleted in the wiki site after user do the 'push'.
>
> Please, avoid long lines (> 80 characters).
>
>> +	open(my $g,"-|","git " . $_[0]);
>
> Space after , please.
>
>> +	my %hashFiles = get_file_extensions_maybe($complete_file_name);
>
> What does this function do? My first understanding was that it 
> queried
> the wiki for allowed file extensions, but why does it need the file
> name? It does nothing if $complete_file_name ends with .mw, but then 
> why
> do you run it before entering the following if() statement?
>
>>  	if (substr($complete_file_name,-3) eq ".mw") {
>>  		my $title = substr($complete_file_name,0,-3);
>
  This function will get a list of allowed file extensions. It need the 
file name to verify if this name begins with .mw or not. If not, it does 
nothing like
you said. But if $complete_file_name is not a wiki page (end with .mw), 
the function will return the list of file extensions %hashFiles to 
verify if this file is allowed in this condition:

                              } elsif (exists($hashFiles{$extension})) {


>> @@ -653,39 +666,74 @@ sub mw_push_file {
>>  			# special priviledges. A common
>>  			# convention is to replace the page
>>  			# with this content instead:
>> -			$file_content = DELETED_CONTENT;
>> +			mw_connect_maybe();
>> +			my $re = $mediawiki->edit( {
>> +				action => 'delete',
>> +				title => $title,
>> +				reason => $summary
>> +				} )|| die $mediawiki-> {error}->{code} . ':' . 
>> $mediawiki->{error}->{details};
>
> This is an unrelated topic, and should not appear in this patch.
>
> If you want to propagate page deletions, then you also need to deal 
> with
> the case where the user is not allowed to do so (very common on
> MediaWiki). Also, if you change the code corresponding to the comment
> right above, you should update the comment too.
>
>> +	elsif (exists($hashFiles{$extension}))
>> +	{
>
Yes, i'll correct it in the next patch.
For the moment, i added these lines to deal with the case similar - the 
case when an user tries to upload a file or pages but he doesn't have 
sufficient rights or he failed to login.

+sub error_insufficient_right {
+       print STDERR "Can not delete or upload file and wiki pages\n";
+       print STDERR "You don't have right to do it\n";
+       print STDOUT "error $_[0]\"right insufficient\"\n";
+       return 0;
+}
@@ -726,17 +1011,32 @@ sub mw_push_file {
                                         ignorewarnings=>1,
                                         }, {
                                                   skip_encoding => 1 # 
Helps with names with accentuated characters
-                                               } ) || die $mediawiki-> 
{error}->{code} . ':' . $mediawiki->{error}->{details};
+                                               } );
+                               if (!$res) {
+                                       if ($mediawiki->{error}->{code} 
== 3) {
+                                               # Failed to upload, 
user didn't login or he doesn't have sufficient rights
+                                               print STDERR 'Warning: 
Error ' .
+                                                   
$mediawiki->{error}->{code} .
+                                                   ' from mediwiki: ' 
. $mediawiki->{error}->{details} .
+                                                   ".\n";
+                                               return 
($newrevid,"insufficient-right");
+                                       } else {
+                                               # Other errors. 
Shouldn't happen => just die()
+                                               die 'Fatal: Error ' .
+                                                   
$mediawiki->{error}->{code} .
+                                                   ' from mediwiki: ' 
. $mediawiki->{error}->{details};
+                                       }
+                               }


@@ -860,6 +1161,9 @@ sub mw_push_revision {
                                 # accurate error message.
                                 return error_non_fast_forward($remote);
                         }
+                       if($status eq "insufficient-right") {
+                               return 
error_insufficient_right($remote);
+                       }
                         if ($status ne "ok") {
                                 die("Unknown error from 
mw_push_file()");
                         }



>
>>  	} else {
>>  		print STDERR "$complete_file_name not a mediawiki file (Not 
>> pushable on this version of git-remote-mediawiki).\n"
>>  	}
>
> Isn't the very point of this patch to remove this error message?
Now, the message is
+               print STDERR "$complete_file_name is not a permitted 
file. Check the configuration of file uploads in your mediawiki \n"
+       }


>
>> @@ -825,3 +873,25 @@ sub mw_push_revision {
>>  	print STDOUT "ok $remote\n";
>>  	return 1;
>>  }
>> +
>> +sub get_file_extensions_maybe {
>> +	my $file_name = shift;
>> +	my $est_mw_page = substr($file_name,-3) eq ".mw";
>
> English please. "est" is french ;-).
Corrected.
-       my $est_mw_page = substr($file_name,-3) eq ".mw";
-       if(!$est_mw_page) {
+       my $is_mw_page = substr($file_name,-3) eq ".mw";
+       if(!$is_mw_page) {

Thanks you so much for your advices.

Thuat.

Re: [PATCHv1] Export file in git-remote-mediawiki

[Matthieu Moy]

nguyenki <nguyenki@ensibm.imag.fr> writes:

>>> +	my %hashFiles = get_file_extensions_maybe($complete_file_name);
>>
>> What does this function do? My first understanding was that it
>> queried
>> the wiki for allowed file extensions, but why does it need the file
>> name? It does nothing if $complete_file_name ends with .mw, but then
>> why
>> do you run it before entering the following if() statement?
>>
>>>  	if (substr($complete_file_name,-3) eq ".mw") {
>>>  		my $title = substr($complete_file_name,0,-3);
>>
>  This function will get a list of allowed file extensions. It need the
> file name to verify if this name begins with .mw or not. If not, it
> does nothing like
> you said. But if $complete_file_name is not a wiki page (end with
> .mw), the function will return the list of file extensions %hashFiles
> to verify if this file is allowed in this condition:

If $complete_file_name is not a wiki page, you basically don't need to
call this function. As I said, move the call to the function within the
next "if" statement, and you won't need this extra complexity.

> Yes, i'll correct it in the next patch.
> For the moment, i added these lines to deal with the case similar -
> the case when an user tries to upload a file or pages but he doesn't
> have sufficient rights or he failed to login.
>
> +sub error_insufficient_right {
> +       print STDERR "Can not delete or upload file and wiki pages\n";
> +       print STDERR "You don't have right to do it\n";
> +       print STDOUT "error $_[0]\"right insufficient\"\n";
> +       return 0;
> +}

This is still a regression. The previous version was propagating page
deletion as "replace content with [[Category:Deleted]]", which did work
with insufficient priviledge.

-- 
Matthieu Moy
http://www-verimag.imag.fr/~moy/